From 7aebaf0e01949067aabe803ad6e01c8ba30c6293 Mon Sep 17 00:00:00 2001 From: John Rushford Date: Wed, 23 Jan 2019 14:25:00 +0000 Subject: [PATCH 1/2] Adds YAML format and parsing for the parent.config file --- configs/Makefile.am | 1 + configs/parent.schema.json | 174 +++++++++ configs/parent.yaml.default | 79 ++++ doc/admin-guide/files/parent.config.en.rst | 18 +- include/tscore/MatcherUtils.h | 3 + iocore/aio/Makefile.am | 3 +- iocore/cache/Makefile.am | 3 +- iocore/dns/Makefile.am | 3 +- iocore/hostdb/Makefile.am | 3 +- mgmt/Makefile.am | 3 +- proxy/ControlBase.cc | 169 ++++++--- proxy/ControlMatcher.cc | 191 +++++++--- proxy/ControlMatcher.h | 8 +- proxy/Makefile.am | 6 +- proxy/ParentSelection.cc | 420 +++++++++++++++------ proxy/ParentSelection.h | 2 + proxy/http/Makefile.am | 3 +- proxy/http/remap/Makefile.am | 3 +- proxy/http2/Makefile.am | 3 +- proxy/shared/Makefile.am | 3 +- src/traffic_crashlog/Makefile.inc | 3 +- src/traffic_ctl/Makefile.inc | 3 +- src/traffic_logcat/Makefile.inc | 3 +- src/traffic_logstats/Makefile.inc | 3 +- src/traffic_manager/Makefile.inc | 3 +- src/traffic_server/Makefile.inc | 3 +- src/tscore/MatcherUtils.cc | 70 ++++ 27 files changed, 932 insertions(+), 254 deletions(-) create mode 100644 configs/parent.schema.json create mode 100644 configs/parent.yaml.default diff --git a/configs/Makefile.am b/configs/Makefile.am index 0e03312be64..1e48ac0db08 100644 --- a/configs/Makefile.am +++ b/configs/Makefile.am @@ -32,6 +32,7 @@ dist_sysconf_DATA = \ ip_allow.config.default \ logging.yaml.default \ parent.config.default \ + parent.yaml.default \ plugin.config.default \ remap.config.default \ socks.config.default \ diff --git a/configs/parent.schema.json b/configs/parent.schema.json new file mode 100644 index 00000000000..1416da4908b --- /dev/null +++ b/configs/parent.schema.json @@ -0,0 +1,174 @@ +{ + "title": "Traffic Server parents configuration", + "description": "parent configuration file structure. Licensed under Apache V2 https://www.apache.org/licenses/LICENSE-2.0", + "type": "object", + "properties": { + "parents": { + "type": "array", + "uniqueItems": true, + "items": { + "type": "object", + "properties": { + "dest_domain": { + "description": "A requested domain name, and its subdomains.", + "type": "string" + }, + "dest_host": { + "description": "A requested hostname.", + "type": "string" + }, + "dest_ip": { + "description": "A requested IP address or range of IP addresses separated by a dash (-)..", + "type": "string" + }, + "url_regex": { + "description": "A regular expression (regex) to be found in a URL.", + "type": "string" + }, + "internal": { + "description": "A boolean value, true or false, specifying if the rule should match (or not match) a transaction originating from an internal API.", + "enum": [ + true, + false + ] + }, + "go_direct": { + "description": "send the request directly to the origin bypassing the parent list.", + "enum": [ + true, + false + ] + }, + "max_simple_retries": { + "description": "The maximum number of retries for a simple retry.", + "type": "number", + "minimum": 1, + "maximum": 5 + }, + "max_unavailable_server_retries": { + "description": "The maximum number of retries for an unavailable server retry.", + "type": "number", + "minimum": 1, + "maximum": 5 + }, + "method": { + "description": "A request URL method", + "type": "string", + "enum": [ + "get", + "post", + "put", + "trace" + ] + }, + "parent_retry": { + "description": "Use one or both of 'simple_retry', or 'unavailable_server_retry'", + "type": "string", + "enum": [ + "simple_retry", + "unavailable_server_retry", + "both" + ] + }, + "parent_is_proxy": { + "description": "The parents listed in parent and secondary_parent are caching proxies, set to false if they are origins", + "enum": [ + true, + false + ] + }, + "port": { + "description": "A requested URL port.", + "type": "number" + }, + "prefix": { + "description": "A prefix in the path part of a URL.", + "type": "string" + }, + "qstring": { + "description": "use or ignore the query string when looking up a parent with consistent_hash", + "type": "string", + "enum": [ + "consider", + "ignore" + ] + }, + "round_robin": { + "description": "select to use one of the parent strategy algorithms, default is 'false'", + "enum": [ + "true", + "false", + "strict", + "consistent_hash", + "latched" + ] + }, + "scheme": { + "description": "A request URL method.", + "type": "string", + "enum": [ + "http", + "https" + ] + }, + "secondary_mode": { + "description": "The secondary parent list mode (see parent.config doc), defaults to 1.", + "type": "number", + "minimum": 1, + "maximum": 3 + }, + "src_ip": { + "description": "A client IP address.", + "type": "string" + }, + "suffix": { + "description": "A file suffix in the URL.", + "type": "string" + }, + "time": { + "description": "A time range, such as 08:00-14:00, during which the parent cache is used to serve requests.", + "type": "string" + }, + "parent": { + "description": "list of primary parents", + "type": "string" + }, + "secondary_parent": { + "description": "list of secondary parents", + "type": "string" + }, + "unavailable_server_retry_responses": { + "description": "When using unavailable_server retry, use this to specify a csv separated list of 5xx resposne codes, default is '503'", + "type": "string" + } + }, + "oneOf": [ + { + "required": [ + "dest_domain" + ] + }, + { + "required": [ + "dest_host" + ] + }, + { + "required": [ + "dest_ip" + ] + }, + { + "required": [ + "url_regex" + ] + } + ] + } + } + }, + "additionalProperties": false, + "required": [ + "parents" + ] +} diff --git a/configs/parent.yaml.default b/configs/parent.yaml.default new file mode 100644 index 00000000000..a3fe06bba1d --- /dev/null +++ b/configs/parent.yaml.default @@ -0,0 +1,79 @@ +# +# parent.yaml +# +# Documentation: +# https://docs.trafficserver.apache.org/en/latest/admin-guide/files/parent.config.en.html +# +# The purpose of this file is to specify the parent proxy for +# specific objects or sets of objects +# +# This is the yaml version of the parent.config. Set 'proxy.config.http.parent_proxy.file' +# to 'parent.yaml' to use a yaml formatted configuration, ie: +# +# CONFIG proxy.config.http.parent_proxy.file STRING parent.yaml +# +# All entries are part of a yaml sequence, both sequence formats are supported, see below. +# +# Each record must include exactly one primary specifier +# +# Primary destination specifiers are +# dest_domain: +# dest_host: +# dest_ip: +# url_regex: +# +# +# record may include any number of the secondary specifiers but +# secondary specifiers may not be duplicated in the same record +# +# Secondary specifiers are +# port: +# scheme: +# prefix: +# suffix: +# method: +# time: +# src_ip: +# internal: {true,false} +# +# Available parent directives are: +# parent: (a semicolon separated list of parent proxies) +# go_direct: {true,false} +# round_robin: {strict,true,false,consistent_hash,latched} +# +# Note: for round_robin, strict means strict round_robin - parents are +# tried one by one, true means round_robin based on client IP +# addresses, false means no round_robin, consisten_hash means a hash +# algorithm over the url path is used, latched is the same as false +# but will stay latched to a selected parent until it is marked down. +# +# Each line must include a parent: directive or a go_direct: +# directive. If both appear, Traffic Server will directly +# contact the origin server if all the listed parent proxies +# are down +# +# Example for two records one in two different yaml sequence formats: +# +# Alternate requests between proxy1 and proxy2 +# +# Sequence Format 1 (2 entries), most readable, dashes matter: +# +# - { dest_domain: "stooges.net", parent: "curly:80|1.0;joe:80|1.0;larry:80|1.0", +# round_robin: "consistent_hash", go_direct: true} +# - { dest_domain: "lucy.net", parent: "ethel:80|1.0;fred:80|1.0;ricky:80|1.0", +# roun_robin: "true", go_direct: false} +# +# Sequence Format 2 (two entries), tabs and dashes matter. +# +# - +# dest_domain: "stooges.net" +# parent: "curly:80|1.0;joe:80|1.0;larry:80|1.0" +# round_robin: "consistent_hash" +# go_direct: true +# - +# dest_domain: "lucy.net" +# parent: "ethel:80|1.0;fred:80|1.0;ricky:80|1.0" +# roun_robin: "true" +# go_direct: false +# +# diff --git a/doc/admin-guide/files/parent.config.en.rst b/doc/admin-guide/files/parent.config.en.rst index b47bf4267bb..354e3de77c3 100644 --- a/doc/admin-guide/files/parent.config.en.rst +++ b/doc/admin-guide/files/parent.config.en.rst @@ -21,24 +21,34 @@ parent.config ============= -.. configfile:: parent.config +.. configfile:: parent.config, parent.yaml -The :file:`parent.config` file identifies the parent proxies used in an +The :file:`parent.config` or optionally `parent.yaml` file identifies the parent proxies used in an cache hierarchy. Use this file to perform the following configuration: - Set up parent cache hierarchies, with multiple parents and parent failover - Configure selected URL requests to bypass parent proxies -Traffic Server uses the :file:`parent.config` file only when the parent +Traffic Server uses the :file:`parent.config` or `parent.yaml` file only when the parent caching option is enabled (refer to :ref:`configuring-traffic-server-to-use-a-parent-cache`). -After you modify the :file:`parent.config` file, run the :option:`traffic_ctl config reload` +After you modify the :file:`parent.config` or `parent.yaml` file, run the :option:`traffic_ctl config reload` command to apply your changes. Format ====== +Optionally the parent configuration may be expressed using a yaml configuration file. +To use `parent.yaml`, set the following in `records.config`:: + + CONFIG proxy.config.http.parent_proxy.file STRING parent.yaml + +When Yaml is used, each line as described here become yaml object which +is part of a sequence. For a YAML example configuration file see:: + + configs/parent.yaml.default + Each line in the :file:`parent.config` file must contain a parent caching rule. Traffic Server recognizes three space-delimited tags: :: diff --git a/include/tscore/MatcherUtils.h b/include/tscore/MatcherUtils.h index 383c77c2c3f..322ea500539 100644 --- a/include/tscore/MatcherUtils.h +++ b/include/tscore/MatcherUtils.h @@ -34,6 +34,7 @@ #include "tscore/ParseRules.h" #include "tscore/Result.h" #include "tscore/ink_inet.h" +#include "yaml-cpp/yaml.h" // Look in MatcherUtils.cc for comments on function usage char *readIntoBuffer(const char *file_path, const char *module_name, int *read_size_ptr); @@ -86,6 +87,7 @@ struct matcher_line { int num_el; // Number of elements char *line[2][MATCHER_MAX_TOKENS]; // label, value pairs int line_num; // config file line number + const YAML::Node *node; // the yaml node if we're parsing yaml matcher_line *next; // use for linked list }; @@ -113,6 +115,7 @@ extern const matcher_tags ip_allow_dest_tags; extern const matcher_tags socks_server_tags; const char *parseConfigLine(char *line, matcher_line *p_line, const matcher_tags *tags); +const char *parseYamlDoc(const YAML::Node node, matcher_line *p_line, const matcher_tags *tags); // inline void LowerCaseStr(char* str) // diff --git a/iocore/aio/Makefile.am b/iocore/aio/Makefile.am index 999f80de6c5..13483dbd37c 100644 --- a/iocore/aio/Makefile.am +++ b/iocore/aio/Makefile.am @@ -20,7 +20,8 @@ AM_CPPFLAGS += \ -I$(abs_top_srcdir)/iocore/eventsystem \ -I$(abs_top_srcdir)/include \ -I$(abs_top_srcdir)/lib \ - $(TS_INCLUDES) + $(TS_INCLUDES) \ + @YAMLCPP_INCLUDES@ TESTS = test_AIO.sample diff --git a/iocore/cache/Makefile.am b/iocore/cache/Makefile.am index e9c9df3a28d..ff2bbb474ab 100644 --- a/iocore/cache/Makefile.am +++ b/iocore/cache/Makefile.am @@ -27,7 +27,8 @@ AM_CPPFLAGS += \ -I$(abs_top_srcdir)/proxy/http/remap \ -I$(abs_top_srcdir)/mgmt \ -I$(abs_top_srcdir)/mgmt/utils \ - $(TS_INCLUDES) + $(TS_INCLUDES) \ + @YAMLCPP_INCLUDES@ noinst_LIBRARIES = libinkcache.a diff --git a/iocore/dns/Makefile.am b/iocore/dns/Makefile.am index a7287d2b295..8507b907728 100644 --- a/iocore/dns/Makefile.am +++ b/iocore/dns/Makefile.am @@ -25,7 +25,8 @@ AM_CPPFLAGS += \ -I$(abs_top_srcdir)/proxy/hdrs \ -I$(abs_top_srcdir)/mgmt \ -I$(abs_top_srcdir)/mgmt/utils \ - $(TS_INCLUDES) + $(TS_INCLUDES) \ + @YAMLCPP_INCLUDES@ noinst_LIBRARIES = libinkdns.a diff --git a/iocore/hostdb/Makefile.am b/iocore/hostdb/Makefile.am index 8f49b12d686..6b2e89daea6 100644 --- a/iocore/hostdb/Makefile.am +++ b/iocore/hostdb/Makefile.am @@ -25,7 +25,8 @@ AM_CPPFLAGS += \ -I$(abs_top_srcdir)/proxy/http \ -I$(abs_top_srcdir)/mgmt \ -I$(abs_top_srcdir)/mgmt/utils \ - $(TS_INCLUDES) + $(TS_INCLUDES) \ + @YAMLCPP_INCLUDES@ EXTRA_DIST = I_HostDB.h diff --git a/mgmt/Makefile.am b/mgmt/Makefile.am index 34c8b031cf0..3d413b1c7dd 100644 --- a/mgmt/Makefile.am +++ b/mgmt/Makefile.am @@ -34,7 +34,8 @@ AM_CPPFLAGS += \ -I$(abs_top_srcdir)/proxy \ -I$(abs_top_srcdir)/proxy/http \ -I$(abs_top_srcdir)/proxy/hdrs \ - $(TS_INCLUDES) + $(TS_INCLUDES) \ + @YAMLCPP_INCLUDES@ libmgmt_c_la_SOURCES = \ RecordsConfig.cc \ diff --git a/proxy/ControlBase.cc b/proxy/ControlBase.cc index 6c3b648b81c..f17630bd11e 100644 --- a/proxy/ControlBase.cc +++ b/proxy/ControlBase.cc @@ -785,8 +785,7 @@ ControlBase::ProcessModifiers(matcher_line *line_info) // Variables for error processing const char *errBuf = nullptr; mod_errors err = ME_UNKNOWN; - - int n_elts = line_info->num_el; // Element count for line. + int n_elts = line_info->num_el; // Element count for line. // No elements -> no modifiers. if (0 >= n_elts) { @@ -796,58 +795,122 @@ ControlBase::ProcessModifiers(matcher_line *line_info) _mods.clear(); _mods.reserve(n_elts); - // As elements are consumed, the labels are nulled out and the element - // count decremented. So we have to scan the entire array to be sure of - // finding all the elements. We'll track the element count so we can - // escape if we've found all of the elements. - for (int i = 0; n_elts && ME_UNKNOWN == err && i < MATCHER_MAX_TOKENS; ++i) { - Modifier *mod = nullptr; - - char *label = line_info->line[0][i]; - char *value = line_info->line[1][i]; - - if (!label) { - continue; // Already use. + if (!line_info->node) { // not yaml + // As elements are consumed, the labels are nulled out and the element + // count decremented. So we have to scan the entire array to be sure of + // finding all the elements. We'll track the element count so we can + // escape if we've found all of the elements. + for (int i = 0; n_elts && ME_UNKNOWN == err && i < MATCHER_MAX_TOKENS; ++i) { + Modifier *mod = nullptr; + + char *label = line_info->line[0][i]; + char *value = line_info->line[1][i]; + + if (!label) { + continue; // Already use. + } + if (!value) { + err = ME_PARSE_FAILED; + break; + } + + if (strcasecmp(label, "port") == 0) { + mod = PortMod::make(value, &errBuf); + } else if (strcasecmp(label, "iport") == 0) { + mod = IPortMod::make(value, &errBuf); + } else if (strcasecmp(label, "scheme") == 0) { + mod = SchemeMod::make(value, &errBuf); + } else if (strcasecmp(label, "method") == 0) { + mod = MethodMod::make(value, &errBuf); + } else if (strcasecmp(label, "prefix") == 0) { + mod = PrefixMod::make(value, &errBuf); + } else if (strcasecmp(label, "suffix") == 0) { + mod = SuffixMod::make(value, &errBuf); + } else if (strcasecmp(label, "src_ip") == 0) { + mod = SrcIPMod::make(value, &errBuf); + } else if (strcasecmp(label, "time") == 0) { + mod = TimeMod::make(value, &errBuf); + } else if (strcasecmp(label, "tag") == 0) { + mod = TagMod::make(value, &errBuf); + } else if (strcasecmp(label, "internal") == 0) { + mod = InternalMod::make(value, &errBuf); + } else { + err = ME_BAD_MOD; + } + + if (errBuf) { + err = ME_CALLEE_GENERATED; // Mod make failed. + } + + // If nothing went wrong, add the mod and bump the element count. + if (ME_UNKNOWN == err) { + _mods.push_back(mod); + --n_elts; + } else { + delete mod; // we still need to clean up because we didn't transfer ownership. + } } - if (!value) { - err = ME_PARSE_FAILED; - break; - } - - if (strcasecmp(label, "port") == 0) { - mod = PortMod::make(value, &errBuf); - } else if (strcasecmp(label, "iport") == 0) { - mod = IPortMod::make(value, &errBuf); - } else if (strcasecmp(label, "scheme") == 0) { - mod = SchemeMod::make(value, &errBuf); - } else if (strcasecmp(label, "method") == 0) { - mod = MethodMod::make(value, &errBuf); - } else if (strcasecmp(label, "prefix") == 0) { - mod = PrefixMod::make(value, &errBuf); - } else if (strcasecmp(label, "suffix") == 0) { - mod = SuffixMod::make(value, &errBuf); - } else if (strcasecmp(label, "src_ip") == 0) { - mod = SrcIPMod::make(value, &errBuf); - } else if (strcasecmp(label, "time") == 0) { - mod = TimeMod::make(value, &errBuf); - } else if (strcasecmp(label, "tag") == 0) { - mod = TagMod::make(value, &errBuf); - } else if (strcasecmp(label, "internal") == 0) { - mod = InternalMod::make(value, &errBuf); - } else { - err = ME_BAD_MOD; - } - - if (errBuf) { - err = ME_CALLEE_GENERATED; // Mod make failed. - } - - // If nothing went wrong, add the mod and bump the element count. - if (ME_UNKNOWN == err) { - _mods.push_back(mod); - --n_elts; - } else { - delete mod; // we still need to clean up because we didn't transfer ownership. + } else { // parsing a yaml config + static constexpr unsigned N_LABEL = 10; + std::array labels = {"port", "iport", "scheme", "method", "prefix", + "suffix", "src_ip", "time", "tag", "internal"}; + ink_assert(line_info->node != nullptr); + + // const YAML::Node root = *(line_info->node); + YAML::Node node = *(line_info->node); + + for (uint i = 0; i < 10; i++) { + Modifier *mod = nullptr; + const char *l = labels[i]; + if (node[l]) { + std::string value = node[l].as(); + char *v = ats_strdup(value.c_str()); + switch (i) { + case 0: // port + mod = PortMod::make(v, &errBuf); + break; + case 1: // iport + mod = IPortMod::make(v, &errBuf); + break; + case 2: // scheme + mod = SchemeMod::make(v, &errBuf); + break; + case 3: // method + mod = MethodMod::make(v, &errBuf); + break; + case 4: // prefix + mod = PrefixMod::make(v, &errBuf); + break; + case 5: // suffix + mod = SuffixMod::make(v, &errBuf); + break; + case 6: // src_ip + mod = SrcIPMod::make(v, &errBuf); + break; + case 7: // time + mod = TimeMod::make(v, &errBuf); + break; + case 8: // tag + mod = TagMod::make(v, &errBuf); + break; + case 9: // internal + mod = InternalMod::make(v, &errBuf); + break; + default: + err = ME_BAD_MOD; + } + + ats_free(v); + v = nullptr; + + // If nothing went wrong, add the mod and bump the element count. + if (ME_UNKNOWN == err) { + _mods.push_back(mod); + --n_elts; + } else { + delete mod; // we still need to clean up because we didn't transfer ownership. + } + } } } diff --git a/proxy/ControlMatcher.cc b/proxy/ControlMatcher.cc index 57c18696690..44298b5ca27 100644 --- a/proxy/ControlMatcher.cc +++ b/proxy/ControlMatcher.cc @@ -671,7 +671,8 @@ IpMatcher::Print() } template -ControlMatcher::ControlMatcher(const char *file_var, const char *name, const matcher_tags *tags, int flags_in) +ControlMatcher::ControlMatcher(const char *file_var, const char *name, const matcher_tags *tags, + const char *_yaml_namespace, int flags_in) { flags = flags_in; ink_assert(flags & (ALLOW_HOST_TABLE | ALLOW_REGEX_TABLE | ALLOW_URL_TABLE | ALLOW_IP_TABLE)); @@ -689,6 +690,12 @@ ControlMatcher::ControlMatcher(const char *file_var, const ch ink_strlcpy(config_file_path, config_path, sizeof(config_file_path)); } + if (file_var && ts::TextView{RecConfigReadConfigPath(file_var)}.take_suffix_at('.') == "yaml") { + config_is_yaml = true; + yaml_namespace = _yaml_namespace; + ink_release_assert(yaml_namespace); + } + reMatch = nullptr; urlMatch = nullptr; hostMatch = nullptr; @@ -783,6 +790,7 @@ ControlMatcher::BuildTableFromString(char *file_buf) int second_pass = 0; int numEntries = 0; bool alarmAlready = false; + const char *errptr = nullptr; // type counts int hostDomain = 0; @@ -791,71 +799,130 @@ ControlMatcher::BuildTableFromString(char *file_buf) int ip = 0; int hostregex = 0; - if (bufTok.Initialize(file_buf, SHARE_TOKS | ALLOW_EMPTY_TOKS) == 0) { - // We have an empty file - return 0; - } - // First get the number of entries - tmp = bufTok.iterFirst(&i_state); - while (tmp != nullptr) { - line_num++; - - // skip all blank spaces at beginning of line - while (*tmp && isspace(*tmp)) { - tmp++; + if (!config_is_yaml) { // parse old line format. + if (bufTok.Initialize(file_buf, SHARE_TOKS | ALLOW_EMPTY_TOKS) == 0) { + // We have an empty file + return 0; } + // build a linked list of the entries + tmp = bufTok.iterFirst(&i_state); + while (tmp != nullptr) { + line_num++; + + // skip all blank spaces at beginning of line + while (*tmp && isspace(*tmp)) { + tmp++; + } - if (*tmp != '#' && *tmp != '\0') { - const char *errptr; - - current = (matcher_line *)ats_malloc(sizeof(matcher_line)); - errptr = parseConfigLine((char *)tmp, current, config_tags); - - if (errptr != nullptr) { - if (config_tags != &socks_server_tags) { - Result error = - Result::failure("%s discarding %s entry at line %d : %s", matcher_name, config_file_path, line_num, errptr); - SignalError(error.message(), alarmAlready); - } - ats_free(current); - } else { - // Line parsed ok. Figure out what the destination - // type is and link it into our list - numEntries++; - current->line_num = line_num; - - switch (current->type) { - case MATCH_HOST: - case MATCH_DOMAIN: - hostDomain++; - break; - case MATCH_IP: - ip++; - break; - case MATCH_REGEX: - regex++; - break; - case MATCH_URL: - url++; - break; - case MATCH_HOST_REGEX: - hostregex++; - break; - case MATCH_NONE: - default: - ink_assert(0); + if (*tmp != '#' && *tmp != '\0') { + current = static_cast(ats_malloc(sizeof(matcher_line))); + errptr = parseConfigLine((char *)tmp, current, config_tags); + + if (errptr != nullptr) { + if (config_tags != &socks_server_tags) { + Result error = + Result::failure("%s discarding %s entry at line %d : %s", matcher_name, config_file_path, line_num, errptr); + SignalError(error.message(), alarmAlready); + } + ats_free(current); + } else { + // Line parsed ok. Figure out what the destination + // type is and link it into our list + numEntries++; + current->line_num = line_num; + + if (first == nullptr) { + ink_assert(last == nullptr); + first = last = current; + } else { + last->next = current; + last = current; + } } + } + tmp = bufTok.iterNext(&i_state); + } + } else { // parse a yaml document and build the linked list of entries.. + /* + * The ControlMatcher has parsed text lines where each line has one 'matcher_tag' with configuration + * parameters specific to the 'Data' type. + * + * This Yaml Parser therefore assumes that the corresponding Yaml document is a sequence of objects + * where each object is a map containing one 'matcher_tag' and it's string value along with the + * 'Data' type specific config parameters. Here we only count the number of entries and the 'matcher_tags'. + * The 'Data' type YamlParser will know how to parse out it's config parameters from each object in the + * sequence when it is called. + * + */ + try { + config = YAML::Load(file_buf); + if (!config.IsMap()) { + Warning("Error parsing the yaml document from the config file %s: expected a top level map document.", config_file_path); + return 1; + } + root = config[yaml_namespace]; + if (!root.IsSequence()) { + Warning("Error parsing the yaml document from the config file %s: expected a sequence document.", config_file_path); + return 1; + } - if (first == nullptr) { - ink_assert(last == nullptr); - first = last = current; + for (uint i = 0; i < root.size(); i++) { + current = static_cast(ats_malloc(sizeof(matcher_line))); + memset(current, 0, sizeof(matcher_line)); + YAML::Node node = root[i]; + current->line_num = i; // overloaded to the yaml node position within the list + errptr = parseYamlDoc(node, current, config_tags); + if (errptr != nullptr) { + if (config_tags != &socks_server_tags) { + Result error = Result::failure("%s discarding %s entry %d : %s", matcher_name, config_file_path, i, errptr); + SignalError(error.message(), alarmAlready); + } + ats_free(current); } else { - last->next = current; - last = current; + // node parsed ok. Figure out what the destination + // type is and link it into our list + numEntries++; + + if (first == nullptr) { + ink_assert(last == nullptr); + first = last = current; + } else { + last->next = current; + last = current; + } } } + } catch (YAML::ParserException &e) { + Warning("Error parsing the yaml config file %s: %s", config_file_path, e.what()); + return 1; + } + } + + // count the entry types + current = first; + while (current != nullptr) { + switch (current->type) { + case MATCH_HOST: + case MATCH_DOMAIN: + hostDomain++; + break; + case MATCH_IP: + ip++; + break; + case MATCH_REGEX: + regex++; + break; + case MATCH_URL: + url++; + break; + case MATCH_HOST_REGEX: + hostregex++; + break; + case MATCH_NONE: + default: + ink_assert(0); } - tmp = bufTok.iterNext(&i_state); + current = current->next; } // Make we have something to do before going on @@ -892,6 +959,12 @@ ControlMatcher::BuildTableFromString(char *file_buf) current = first; while (current != nullptr) { Result error = Result::ok(); + if (config_is_yaml) { + YAML::Node node = root[current->line_num]; + current->node = &node; + } else { + current->node = nullptr; + } second_pass++; if ((flags & ALLOW_HOST_TABLE) && current->type == MATCH_DOMAIN) { @@ -946,7 +1019,9 @@ ControlMatcher::BuildTable() } ret = BuildTableFromString(file_buf); + ats_free(file_buf); + return ret; } diff --git a/proxy/ControlMatcher.h b/proxy/ControlMatcher.h index 50cb2916e6e..1509dfab6f4 100644 --- a/proxy/ControlMatcher.h +++ b/proxy/ControlMatcher.h @@ -92,6 +92,8 @@ #include "tscore/ink_apidefs.h" #include "tscore/ink_defs.h" +#include "tscore/ts_file.h" +#include "tscpp/util/TextView.h" #include "HTTP.h" #include "tscore/Regex.h" #include "URL.h" @@ -301,7 +303,7 @@ template class ControlMatcher public: // Parameter name must not be deallocated before this // object is - ControlMatcher(const char *file_var, const char *name, const matcher_tags *tags, + ControlMatcher(const char *file_var, const char *name, const matcher_tags *tags, const char *_yaml_namespace = nullptr, int flags_in = (ALLOW_HOST_TABLE | ALLOW_IP_TABLE | ALLOW_REGEX_TABLE | ALLOW_HOST_REGEX_TABLE | ALLOW_URL_TABLE)); ~ControlMatcher(); int BuildTable(); @@ -357,4 +359,8 @@ template class ControlMatcher int flags = 0; int m_numEntries = 0; const char *matcher_name = "unknown"; // Used for Debug/Warning/Error messages + bool config_is_yaml = false; // config file is in yaml format. + YAML::Node config; + YAML::Node root; + const char *yaml_namespace = nullptr; // config file top level yaml namespace. }; diff --git a/proxy/Makefile.am b/proxy/Makefile.am index 1965837408d..f1482a0c41f 100644 --- a/proxy/Makefile.am +++ b/proxy/Makefile.am @@ -32,8 +32,9 @@ AM_CPPFLAGS += \ -I$(abs_srcdir)/hdrs \ -I$(abs_top_srcdir)/mgmt \ -I$(abs_top_srcdir)/mgmt/utils \ - $(TS_INCLUDES) - + $(TS_INCLUDES) \ + @YAMLCPP_INCLUDES@ + noinst_HEADERS = \ Show.h @@ -76,7 +77,6 @@ libproxy_a_SOURCES += \ RegressionSM.cc endif - # These are currently built separate, as part of building the lib/ tree, using # the normal LuaJIT build system. We are using the .o's directly, instead of the # luajit.a to avoid the linker from optimizing symbols away. We could maybe diff --git a/proxy/ParentSelection.cc b/proxy/ParentSelection.cc index a529797b37d..89a7f5c0a4b 100644 --- a/proxy/ParentSelection.cc +++ b/proxy/ParentSelection.cc @@ -30,6 +30,7 @@ #include "HTTP.h" #include "HttpTransact.h" #include "I_Machine.h" +#include "yaml-cpp/yaml.h" #define MAX_SIMPLE_RETRIES 5 #define MAX_UNAVAILABLE_SERVER_RETRIES 5 @@ -38,6 +39,7 @@ typedef ControlMatcher P_table; // Global Vars for Parent Selection static const char modulePrefix[] = "[ParentSelection]"; +static const char yaml_namespace[] = "parents"; static ConfigUpdateHandler *parentConfigUpdate = nullptr; static int self_detect = 2; @@ -269,7 +271,7 @@ ParentConfig::reconfigure() ParentConfigParams *params = nullptr; // Allocate parent table - P_table *pTable = new P_table(file_var, modulePrefix, &http_dest_tags); + P_table *pTable = new P_table(file_var, modulePrefix, &http_dest_tags, yaml_namespace); params = new ParentConfigParams(pTable); ink_assert(params != nullptr); @@ -606,141 +608,269 @@ ParentRecord::Init(matcher_line *line_info) char buf[128]; RecInt rec_self_detect = 2; - this->line_num = line_info->line_num; - this->scheme = nullptr; - if (RecGetRecordInt("proxy.config.http.parent_proxy.self_detect", &rec_self_detect) == REC_ERR_OKAY) { self_detect = static_cast(rec_self_detect); } - for (int i = 0; i < MATCHER_MAX_TOKENS; i++) { - used = false; - label = line_info->line[0][i]; - val = line_info->line[1][i]; + if (!line_info->node) { // not yaml, parse text line . + + this->line_num = line_info->line_num; + this->scheme = nullptr; + + for (int i = 0; i < MATCHER_MAX_TOKENS; i++) { + used = false; + label = line_info->line[0][i]; + val = line_info->line[1][i]; + + if (label == nullptr) { + continue; + } + + if (strcasecmp(label, "round_robin") == 0) { + if (strcasecmp(val, "true") == 0) { + round_robin = P_HASH_ROUND_ROBIN; + } else if (strcasecmp(val, "strict") == 0) { + round_robin = P_STRICT_ROUND_ROBIN; + } else if (strcasecmp(val, "false") == 0) { + round_robin = P_NO_ROUND_ROBIN; + } else if (strcasecmp(val, "consistent_hash") == 0) { + round_robin = P_CONSISTENT_HASH; + } else if (strcasecmp(val, "latched") == 0) { + round_robin = P_LATCHED_ROUND_ROBIN; + } else { + round_robin = P_NO_ROUND_ROBIN; + errPtr = "invalid argument to round_robin directive"; + } + used = true; + } else if (strcasecmp(label, "parent") == 0 || strcasecmp(label, "primary_parent") == 0) { + PreProcessParents(val, line_num, parent_buf, sizeof(parent_buf) - 1); + errPtr = ProcessParents(parent_buf, true); + used = true; + } else if (strcasecmp(label, "secondary_parent") == 0) { + PreProcessParents(val, line_num, parent_buf, sizeof(parent_buf) - 1); + errPtr = ProcessParents(parent_buf, false); + used = true; + } else if (strcasecmp(label, "go_direct") == 0) { + if (strcasecmp(val, "false") == 0) { + go_direct = false; + } else if (strcasecmp(val, "true") != 0) { + errPtr = "invalid argument to go_direct directive"; + } else { + go_direct = true; + } + used = true; + } else if (strcasecmp(label, "qstring") == 0) { + // qstring=ignore | consider + if (strcasecmp(val, "ignore") == 0) { + this->ignore_query = true; + } else { + this->ignore_query = false; + } + used = true; + } else if (strcasecmp(label, "parent_is_proxy") == 0) { + if (strcasecmp(val, "false") == 0) { + parent_is_proxy = false; + } else if (strcasecmp(val, "true") != 0) { + errPtr = "invalid argument to parent_is_proxy directive"; + } else { + parent_is_proxy = true; + } + used = true; + } else if (strcasecmp(label, "parent_retry") == 0) { + if (strcasecmp(val, "simple_retry") == 0) { + parent_retry = PARENT_RETRY_SIMPLE; + } else if (strcasecmp(val, "unavailable_server_retry") == 0) { + parent_retry = PARENT_RETRY_UNAVAILABLE_SERVER; + } else if (strcasecmp(val, "both") == 0) { + parent_retry = PARENT_RETRY_BOTH; + } else { + errPtr = "invalid argument to parent_retry directive."; + } + used = true; + } else if (strcasecmp(label, "unavailable_server_retry_responses") == 0 && unavailable_server_retry_responses == nullptr) { + unavailable_server_retry_responses = new UnavailableServerResponseCodes(val); + used = true; + } else if (strcasecmp(label, "max_simple_retries") == 0) { + int v = atoi(val); + if (v >= 1 && v < MAX_SIMPLE_RETRIES) { + max_simple_retries = v; + used = true; + } else { + snprintf(buf, sizeof(buf), "invalid argument to max_simple_retries. Argument must be between 1 and %d.", + MAX_SIMPLE_RETRIES); + errPtr = buf; + } + } else if (strcasecmp(label, "max_unavailable_server_retries") == 0) { + int v = atoi(val); + if (v >= 1 && v < MAX_UNAVAILABLE_SERVER_RETRIES) { + max_unavailable_server_retries = v; + used = true; + } else { + snprintf(buf, sizeof(buf), "invalid argument to max_unavailable_server_retries. Argument must be between 1 and %d.", + MAX_UNAVAILABLE_SERVER_RETRIES); + errPtr = buf; + } + } else if (strcasecmp(label, "secondary_mode") == 0) { + int v = atoi(val); + secondary_mode = v; + used = true; + } else if (strcasecmp(label, "ignore_self_detect") == 0) { + if (strcasecmp(val, "true") == 0) { + ignore_self_detect = true; + } else { + ignore_self_detect = false; + } + used = true; + } + // Report errors generated by ProcessParents(); + if (errPtr != nullptr) { + return Result::failure("%s %s at line %d", modulePrefix, errPtr, line_num); + } + + if (used == true) { + // Consume the label/value pair we used + line_info->line[0][i] = nullptr; + line_info->num_el--; + } + } - if (label == nullptr) { - continue; + // delete unavailable_server_retry_responses if unavailable_server_retry is not enabled. + if (unavailable_server_retry_responses != nullptr && !(parent_retry & PARENT_RETRY_UNAVAILABLE_SERVER)) { + Warning("%s ignoring unavailable_server_retry_responses directive on line %d, as unavailable_server_retry is not enabled.", + modulePrefix, line_num); + delete unavailable_server_retry_responses; + unavailable_server_retry_responses = nullptr; + } else if (unavailable_server_retry_responses == nullptr && (parent_retry & PARENT_RETRY_UNAVAILABLE_SERVER)) { + // initialize UnavailableServerResponseCodes to the default value if unavailable_server_retry is enabled. + Warning("%s initializing UnavailableServerResponseCodes on line %d to 503 default.", modulePrefix, line_num); + unavailable_server_retry_responses = new UnavailableServerResponseCodes(nullptr); } - if (strcasecmp(label, "round_robin") == 0) { - if (strcasecmp(val, "true") == 0) { - round_robin = P_HASH_ROUND_ROBIN; - } else if (strcasecmp(val, "strict") == 0) { - round_robin = P_STRICT_ROUND_ROBIN; - } else if (strcasecmp(val, "false") == 0) { - round_robin = P_NO_ROUND_ROBIN; - } else if (strcasecmp(val, "consistent_hash") == 0) { - round_robin = P_CONSISTENT_HASH; - } else if (strcasecmp(val, "latched") == 0) { - round_robin = P_LATCHED_ROUND_ROBIN; - } else { - round_robin = P_NO_ROUND_ROBIN; - errPtr = "invalid argument to round_robin directive"; + // end of text line parsing + + } else { // parse a yaml config. + ink_assert(line_info->node != nullptr); + // const YAML::Node root = *(line_info->node); + YAML::Node node = *(line_info->node); + + try { + if (node["round_robin"]) { + if (node["round_robin"].Scalar().compare("true") == 0) { + round_robin = P_HASH_ROUND_ROBIN; + } else if (node["round_robin"].Scalar().compare("strict") == 0) { + round_robin = P_STRICT_ROUND_ROBIN; + } else if (node["round_robin"].Scalar().compare("false") == 0) { + round_robin = P_NO_ROUND_ROBIN; + } else if (node["round_robin"].Scalar().compare("consistent_hash") == 0) { + round_robin = P_CONSISTENT_HASH; + } else if (node["round_robin"].Scalar().compare("latched") == 0) { + round_robin = P_LATCHED_ROUND_ROBIN; + } else { + errPtr = "invalid argument to round_robin directive"; + return Result::failure("%s %s on entry %d", modulePrefix, errPtr, line_info->line_num); + } } - used = true; - } else if (strcasecmp(label, "parent") == 0 || strcasecmp(label, "primary_parent") == 0) { - PreProcessParents(val, line_num, parent_buf, sizeof(parent_buf) - 1); - errPtr = ProcessParents(parent_buf, true); - used = true; - } else if (strcasecmp(label, "secondary_parent") == 0) { - PreProcessParents(val, line_num, parent_buf, sizeof(parent_buf) - 1); - errPtr = ProcessParents(parent_buf, false); - used = true; - } else if (strcasecmp(label, "go_direct") == 0) { - if (strcasecmp(val, "false") == 0) { - go_direct = false; - } else if (strcasecmp(val, "true") != 0) { - errPtr = "invalid argument to go_direct directive"; - } else { - go_direct = true; + + if (node["parent"] || node["primary_parent"]) { + const char *parent = node["parent"].Scalar().c_str(); + PreProcessParents(parent, line_info->line_num, parent_buf, sizeof(parent_buf) - 1); + errPtr = ProcessParents(parent_buf, true); + if (errPtr != nullptr) { + return Result::failure("%s %s on entry %d", modulePrefix, errPtr, line_info->line_num); + } } - used = true; - } else if (strcasecmp(label, "qstring") == 0) { - // qstring=ignore | consider - if (strcasecmp(val, "ignore") == 0) { - this->ignore_query = true; - } else { - this->ignore_query = false; + + if (node["secondary_parent"]) { + const char *secondary_parent = node["secondary_parent"].Scalar().c_str(); + PreProcessParents(secondary_parent, line_info->line_num, parent_buf, sizeof(parent_buf) - 1); + errPtr = ProcessParents(parent_buf, false); + if (errPtr != nullptr) { + return Result::failure("%s %s on entry %d", modulePrefix, errPtr, line_info->line_num); + } } - used = true; - } else if (strcasecmp(label, "parent_is_proxy") == 0) { - if (strcasecmp(val, "false") == 0) { - parent_is_proxy = false; - } else if (strcasecmp(val, "true") != 0) { - errPtr = "invalid argument to parent_is_proxy directive"; - } else { - parent_is_proxy = true; + + if (node["secondary_mode"]) { + uint sm = node["secondary_mode"].as(); + if (sm >= 1 && sm < 4) { + secondary_mode = sm; + } else { + snprintf(buf, sizeof(buf), "entry %d, invalid argument to secondary_mode. Argument must be between 1, 2 or 3.", + line_num); + Warning("%s %s: using default value %d", modulePrefix, buf, secondary_mode); + } } - used = true; - } else if (strcasecmp(label, "parent_retry") == 0) { - if (strcasecmp(val, "simple_retry") == 0) { - parent_retry = PARENT_RETRY_SIMPLE; - } else if (strcasecmp(val, "unavailable_server_retry") == 0) { - parent_retry = PARENT_RETRY_UNAVAILABLE_SERVER; - } else if (strcasecmp(val, "both") == 0) { - parent_retry = PARENT_RETRY_BOTH; - } else { - errPtr = "invalid argument to parent_retry directive."; + + if (node["go_direct"]) { + go_direct = node["go_direct"].as(); } - used = true; - } else if (strcasecmp(label, "unavailable_server_retry_responses") == 0 && unavailable_server_retry_responses == nullptr) { - unavailable_server_retry_responses = new UnavailableServerResponseCodes(val); - used = true; - } else if (strcasecmp(label, "max_simple_retries") == 0) { - int v = atoi(val); - if (v >= 1 && v < MAX_SIMPLE_RETRIES) { - max_simple_retries = v; - used = true; - } else { - snprintf(buf, sizeof(buf), "invalid argument to max_simple_retries. Argument must be between 1 and %d.", - MAX_SIMPLE_RETRIES); - errPtr = buf; + + if (node["qstring"]) { + if (node["qstring"].Scalar().compare("consider") == 0) { + ignore_query = false; + } else if (node["qstring"].Scalar().compare("ignore") == 0) { + ignore_query = true; + } else { + errPtr = "invalid argument to qstring directive"; + return Result::failure("%s %s on entry %d", modulePrefix, errPtr, line_info->line_num); + } } - } else if (strcasecmp(label, "max_unavailable_server_retries") == 0) { - int v = atoi(val); - if (v >= 1 && v < MAX_UNAVAILABLE_SERVER_RETRIES) { - max_unavailable_server_retries = v; - used = true; - } else { - snprintf(buf, sizeof(buf), "invalid argument to max_unavailable_server_retries. Argument must be between 1 and %d.", - MAX_UNAVAILABLE_SERVER_RETRIES); - errPtr = buf; + + if (node["parent_is_proxy"]) { + parent_is_proxy = node["parent_is_proxy"].as(); } - } else if (strcasecmp(label, "secondary_mode") == 0) { - int v = atoi(val); - secondary_mode = v; - used = true; - } else if (strcasecmp(label, "ignore_self_detect") == 0) { - if (strcasecmp(val, "true") == 0) { - ignore_self_detect = true; - } else { - ignore_self_detect = false; + + if (node["ignore_self_detect"]) { + ignore_self_detect = node["ignore_self_detect"].as(); } - used = true; - } - // Report errors generated by ProcessParents(); - if (errPtr != nullptr) { - return Result::failure("%s %s at line %d", modulePrefix, errPtr, line_num); - } - if (used == true) { - // Consume the label/value pair we used - line_info->line[0][i] = nullptr; - line_info->num_el--; + if (node["parent_retry"]) { + if (node["parent_retry"].Scalar().compare("simple_retry") == 0) { + parent_retry = PARENT_RETRY_SIMPLE; + } else if (node["parent_retry"].Scalar().compare("unavailable_server_retry") == 0) { + parent_retry = PARENT_RETRY_UNAVAILABLE_SERVER; + } else if (node["parent_retry"].Scalar().compare("both") == 0) { + parent_retry = PARENT_RETRY_BOTH; + } else { + errPtr = "invalid argument to simple_retry directive."; + return Result::failure("%s %s on entry %d", modulePrefix, errPtr, line_info->line_num); + } + } + if (parent_retry & PARENT_RETRY_UNAVAILABLE_SERVER) { + if (node["unavailable_server_retry_responses"]) { + const char *usr = node["unavailable_server_retry_responses"].Scalar().c_str(); + unavailable_server_retry_responses = new UnavailableServerResponseCodes(const_cast(usr)); + } else { + unavailable_server_retry_responses = new UnavailableServerResponseCodes(nullptr); + Warning("%s initializing UnavailableServerResponseCodes on entry %d to 503 default.", modulePrefix, line_num); + } + if (node["max_unavailable_server_retries"]) { + uint mus = node["max_unavailable_server_retries"].as(); + if (mus > 0 && mus < MAX_UNAVAILABLE_SERVER_RETRIES) { + max_unavailable_server_retries = mus; + } else { + snprintf(buf, sizeof(buf), + "entry %d, invalid argument to max_unavailable_server_retries. Argument must be between 1 and %d.", line_num, + MAX_UNAVAILABLE_SERVER_RETRIES); + Warning("%s %s: using default value %d", modulePrefix, buf, max_unavailable_server_retries); + } + } + } + if (parent_retry & PARENT_RETRY_SIMPLE) { + if (node["max_simple_retries"]) { + uint msr = node["max_simple_retries"].as(); + if (msr > 0 && msr < MAX_SIMPLE_RETRIES) { + max_simple_retries = msr; + } else { + snprintf(buf, sizeof(buf), "entry %d, invalid argument to max_simple_retries. Argument must be between 1 and %d.", + line_num, MAX_SIMPLE_RETRIES); + Warning("%s %s: using default value %d", modulePrefix, buf, max_simple_retries); + } + } + } + } catch (std::exception &ex) { + return Result::failure("%s parent config yaml parse error at line %d: %s", modulePrefix, line_num, ex.what()); } - } - - // delete unavailable_server_retry_responses if unavailable_server_retry is not enabled. - if (unavailable_server_retry_responses != nullptr && !(parent_retry & PARENT_RETRY_UNAVAILABLE_SERVER)) { - Warning("%s ignoring unavailable_server_retry_responses directive on line %d, as unavailable_server_retry is not enabled.", - modulePrefix, line_num); - delete unavailable_server_retry_responses; - unavailable_server_retry_responses = nullptr; - } else if (unavailable_server_retry_responses == nullptr && (parent_retry & PARENT_RETRY_UNAVAILABLE_SERVER)) { - // initialize UnavailableServerResponseCodes to the default value if unavailable_server_retry is enabled. - Warning("%s initializing UnavailableServerResponseCodes on line %d to 503 default.", modulePrefix, line_num); - unavailable_server_retry_responses = new UnavailableServerResponseCodes(nullptr); - } + } // end of yaml parse. if (this->parents == nullptr && go_direct == false) { return Result::failure("%s No parent specified in parent.config at line %d", modulePrefix, line_num); @@ -1008,15 +1138,36 @@ EXCLUSIVE_REGRESSION_TEST(PARENTSELECTION)(RegressionTest * /* t ATS_UNUSED */, #define REBUILD \ do { \ + static char debug[] = {"parent_select|host_statuses"}; \ delete params; \ - ParentTable = new P_table("", "ParentSelection Unit Test Table", &http_dest_tags, \ + ParentTable = new P_table("", "ParentSelection Unit Test Table", &http_dest_tags, nullptr, \ ALLOW_HOST_TABLE | ALLOW_REGEX_TABLE | ALLOW_URL_TABLE | ALLOW_IP_TABLE | DONT_BUILD_TABLE); \ ParentTable->BuildTableFromString(tbl); \ RecSetRecordInt("proxy.config.http.parent_proxy.fail_threshold", fail_threshold, REC_SOURCE_DEFAULT); \ RecSetRecordInt("proxy.config.http.parent_proxy.retry_time", retry_time, REC_SOURCE_DEFAULT); \ + RecSetRecordInt("proxy.config.diags.debug.enabled", 1, REC_SOURCE_DEFAULT); \ + RecSetRecordString("proxy.config.diags.debug.tags", debug, REC_SOURCE_DEFAULT, true); \ params = new ParentConfigParams(ParentTable); \ } while (0) +// set's the filename with a recognized yaml suffix for testing a yaml config. +#define REBUILD_YAML \ + do { \ + static char debug[] = {"parent_select|host_statuses"}; \ + delete params; \ + const RecString fn = ats_strdup("parent.yaml"); \ + RecSetRecordString("proxy.config.http.parent_proxy.file", fn, REC_SOURCE_DEFAULT, false, false); \ + ParentTable = \ + new P_table("proxy.config.http.parent_proxy.file", "ParentSelection Unit Test Table", &http_dest_tags, "parents", \ + ALLOW_HOST_TABLE | ALLOW_REGEX_TABLE | ALLOW_URL_TABLE | ALLOW_IP_TABLE | DONT_BUILD_TABLE); \ + ParentTable->BuildTableFromString(tbl); \ + RecSetRecordInt("proxy.config.http.parent_proxy.fail_threshold", fail_threshold, REC_SOURCE_DEFAULT); \ + RecSetRecordInt("proxy.config.http.parent_proxy.retry_time", retry_time, REC_SOURCE_DEFAULT); \ + RecSetRecordInt("proxy.config.diags.debug.enabled", 1, REC_SOURCE_DEFAULT); \ + RecSetRecordString("proxy.config.diags.debug.tags", debug, REC_SOURCE_DEFAULT, true); \ + params = new ParentConfigParams(ParentTable); \ + } while (0) + #define REINIT \ do { \ if (request != NULL) { \ @@ -1781,6 +1932,35 @@ EXCLUSIVE_REGRESSION_TEST(PARENTSELECTION)(RegressionTest * /* t ATS_UNUSED */, FP; RE(verify(result, PARENT_SPECIFIED, "carol", 80), 211); + // Test 212 - parse and test a yaml config + _st.setHostStatus("curly", HOST_STATUS_UP, 0, Reason::MANUAL); + tbl[0] = '\0'; + ST(212); + T("parents:\n" + " - { dest_domain: \"stooges.net\", parent: \"curly:80|1.0;joe:8080|1.0;larry:80|1.0\", " + "round_robin: \"true\", go_direct: true, qstring: \"consider\" }\n" + " - { dest_domain: \"bad.net\", parent: \"foo:80|1.0;baz:8080|1.0;bar:80|1.0\", " + "round_robin: \"true\", go_direct: true, qstring: \"consider\" }"); + REBUILD_YAML; + REINIT; + br(request, "i.am.stooges.net"); + FP; + RE(verify(result, PARENT_SPECIFIED, "curly", 80), 212); + + // Test 213 + tbl[0] = '\0'; + ST(213); + T("parents:\n" + " - { dest_domain: \"stooges.net\", parent: \"curly:80|1.0;joe:8080|1.0;larry:80|1.0\", " + "round_robin: \"true\", go_direct: true, qstring: \"consider\" }\n" + " - { dest_domain: \"bad.net\", parent: \"foo:80|1.0;baz:8080|1.0;bar:80|1.0\", " + "round_robin: \"true\", go_direct: true, qstring: \"consider\" }"); + REBUILD_YAML; + REINIT; + br(request, "i.am.bad.net"); + FP; + RE(verify(result, PARENT_SPECIFIED, "foo", 80), 213); + delete request; delete result; delete params; diff --git a/proxy/ParentSelection.h b/proxy/ParentSelection.h index 759a23eaf4a..1a9eba27e8e 100644 --- a/proxy/ParentSelection.h +++ b/proxy/ParentSelection.h @@ -37,7 +37,9 @@ #include "tscore/ConsistentHash.h" #include "tscore/Tokenizer.h" #include "tscore/ink_apidefs.h" +#include "tscore/ts_file.h" #include "HostStatus.h" +#include "yaml-cpp/yaml.h" #include #include diff --git a/proxy/http/Makefile.am b/proxy/http/Makefile.am index 6f5733229b0..7fdb8588591 100644 --- a/proxy/http/Makefile.am +++ b/proxy/http/Makefile.am @@ -32,7 +32,8 @@ AM_CPPFLAGS += \ -I$(abs_top_srcdir)/proxy/http/remap \ -I$(abs_top_srcdir)/proxy/logging \ -I$(abs_top_srcdir)/proxy/http2 \ - $(TS_INCLUDES) + $(TS_INCLUDES) \ + @YAMLCPP_INCLUDES@ noinst_HEADERS = HttpProxyServerMain.h noinst_LIBRARIES = libhttp.a diff --git a/proxy/http/remap/Makefile.am b/proxy/http/remap/Makefile.am index ddee9dcc87b..4a71b3249ec 100644 --- a/proxy/http/remap/Makefile.am +++ b/proxy/http/remap/Makefile.am @@ -28,7 +28,8 @@ AM_CPPFLAGS += \ -I$(abs_top_srcdir)/proxy/hdrs \ -I$(abs_top_srcdir)/proxy/shared \ -I$(abs_top_srcdir)/proxy/http \ - $(TS_INCLUDES) + $(TS_INCLUDES) \ + @YAMLCPP_INCLUDES@ noinst_LIBRARIES = libhttp_remap.a diff --git a/proxy/http2/Makefile.am b/proxy/http2/Makefile.am index 47cbb8e7577..ef362cc6a72 100644 --- a/proxy/http2/Makefile.am +++ b/proxy/http2/Makefile.am @@ -29,7 +29,8 @@ AM_CPPFLAGS += \ -I$(abs_top_srcdir)/proxy/hdrs \ -I$(abs_top_srcdir)/proxy/shared \ -I$(abs_top_srcdir)/proxy/http/remap \ - $(TS_INCLUDES) + $(TS_INCLUDES) \ + @YAMLCPP_INCLUDES@ noinst_LIBRARIES = libhttp2.a diff --git a/proxy/shared/Makefile.am b/proxy/shared/Makefile.am index 1d54d408f10..48a11094ff0 100644 --- a/proxy/shared/Makefile.am +++ b/proxy/shared/Makefile.am @@ -35,7 +35,8 @@ AM_CPPFLAGS += \ -I$(abs_top_srcdir)/proxy/http \ -I$(abs_top_srcdir)/proxy/hdrs \ -I$(abs_top_srcdir)/proxy/logging \ - $(TS_INCLUDES) + $(TS_INCLUDES) \ + @YAMLCPP_INCLUDES@ libdiagsconfig_a_SOURCES = \ DiagsConfig.cc diff --git a/src/traffic_crashlog/Makefile.inc b/src/traffic_crashlog/Makefile.inc index f665ab63d06..86c578130f3 100644 --- a/src/traffic_crashlog/Makefile.inc +++ b/src/traffic_crashlog/Makefile.inc @@ -25,7 +25,8 @@ traffic_crashlog_traffic_crashlog_CPPFLAGS = \ -I$(abs_top_srcdir)/mgmt \ -I$(abs_top_srcdir)/mgmt/utils \ -I$(abs_top_srcdir)/mgmt/api/include \ - $(TS_INCLUDES) + $(TS_INCLUDES) \ + @YAMLCPP_INCLUDES@ traffic_crashlog_traffic_crashlog_LDFLAGS = \ $(AM_LDFLAGS) \ diff --git a/src/traffic_ctl/Makefile.inc b/src/traffic_ctl/Makefile.inc index 446e56539f9..bd920e8c39b 100644 --- a/src/traffic_ctl/Makefile.inc +++ b/src/traffic_ctl/Makefile.inc @@ -27,7 +27,8 @@ traffic_ctl_traffic_ctl_CPPFLAGS = \ -I$(abs_top_srcdir)/mgmt \ -I$(abs_top_srcdir)/mgmt/api/include \ -I$(abs_top_srcdir)/proxy \ - $(TS_INCLUDES) + $(TS_INCLUDES) \ + @YAMLCPP_INCLUDES@ traffic_ctl_traffic_ctl_SOURCES = \ traffic_ctl/alarm.cc \ diff --git a/src/traffic_logcat/Makefile.inc b/src/traffic_logcat/Makefile.inc index fe77562a4c6..99ede49f589 100644 --- a/src/traffic_logcat/Makefile.inc +++ b/src/traffic_logcat/Makefile.inc @@ -29,7 +29,8 @@ traffic_logcat_traffic_logcat_CPPFLAGS = \ -I$(abs_top_srcdir)/proxy/shared \ -I$(abs_top_srcdir)/mgmt \ -I$(abs_top_srcdir)/mgmt/utils \ - $(TS_INCLUDES) + $(TS_INCLUDES) \ + @YAMLCPP_INCLUDES@ traffic_logcat_traffic_logcat_LDFLAGS = \ $(AM_LDFLAGS) \ diff --git a/src/traffic_logstats/Makefile.inc b/src/traffic_logstats/Makefile.inc index 12f8335692f..f16cc856cf6 100644 --- a/src/traffic_logstats/Makefile.inc +++ b/src/traffic_logstats/Makefile.inc @@ -29,7 +29,8 @@ traffic_logstats_traffic_logstats_CPPFLAGS = \ -I$(abs_top_srcdir)/proxy/shared \ -I$(abs_top_srcdir)/mgmt \ -I$(abs_top_srcdir)/mgmt/utils \ - $(TS_INCLUDES) + $(TS_INCLUDES) \ + @YAMLCPP_INCLUDES@ traffic_logstats_traffic_logstats_LDFLAGS = \ $(AM_LDFLAGS) \ diff --git a/src/traffic_manager/Makefile.inc b/src/traffic_manager/Makefile.inc index 19dfb927568..12eb74f487a 100644 --- a/src/traffic_manager/Makefile.inc +++ b/src/traffic_manager/Makefile.inc @@ -28,7 +28,8 @@ traffic_manager_traffic_manager_CPPFLAGS = \ -I$(abs_top_srcdir)/mgmt/api \ -I$(abs_top_srcdir)/mgmt/api/include \ -I$(abs_top_srcdir)/mgmt/utils \ - $(TS_INCLUDES) + $(TS_INCLUDES) \ + @YAMLCPP_INCLUDES@ traffic_manager_traffic_manager_LDFLAGS = \ $(AM_LDFLAGS) \ diff --git a/src/traffic_server/Makefile.inc b/src/traffic_server/Makefile.inc index a18d6cec51d..51942d5cd18 100644 --- a/src/traffic_server/Makefile.inc +++ b/src/traffic_server/Makefile.inc @@ -34,7 +34,8 @@ traffic_server_traffic_server_CPPFLAGS = \ -I$(abs_top_srcdir)/mgmt \ -I$(abs_top_srcdir)/mgmt/utils \ $(TS_INCLUDES) \ - @OPENSSL_INCLUDES@ + @OPENSSL_INCLUDES@ \ + @YAMLCPP_INCLUDES@ traffic_server_traffic_server_LDFLAGS = \ $(AM_LDFLAGS) \ diff --git a/src/tscore/MatcherUtils.cc b/src/tscore/MatcherUtils.cc index cb9822a2d75..c9697f6101d 100644 --- a/src/tscore/MatcherUtils.cc +++ b/src/tscore/MatcherUtils.cc @@ -618,3 +618,73 @@ parseConfigLine(char *line, matcher_line *p_line, const matcher_tags *tags) return nullptr; } + +// char* parseYamlDoc(YAML::Node node, matcher_line* p_line, const matcher_tags* tags) +// +// Parse out a yaml node object suitable for passing to +// a ControlMatcher object +// +// If successful, nullptr is returned. If unsuccessful, +// a static error string is returned +// +const char * +parseYamlDoc(const YAML::Node node, matcher_line *p, const matcher_tags *tags) +{ + const char *label; + std::string_view val; + matcher_type type = MATCH_NONE; + + uint dest = 0; + if (node[tags->match_ip]) { + type = MATCH_IP; + label = tags->match_ip; + val = node[tags->match_ip].Scalar(); + dest++; + } else if (node[tags->match_host]) { + type = MATCH_HOST; + label = tags->match_host; + val = node[tags->match_host].Scalar(); + dest++; + } else if (node[tags->match_domain]) { + type = MATCH_DOMAIN; + label = tags->match_domain; + val = node[tags->match_domain].Scalar(); + dest++; + } else if (node[tags->match_regex]) { + type = MATCH_REGEX; + label = tags->match_regex; + val = node[tags->match_regex].Scalar(); + dest++; + } else if (node[tags->match_url]) { + type = MATCH_URL; + label = tags->match_url; + val = node[tags->match_url].Scalar(); + dest++; + } else if (node[tags->match_host_regex]) { + type = MATCH_HOST_REGEX; + label = tags->match_host_regex; + val = node[tags->match_host_regex].Scalar(); + dest++; + } + + if (type == MATCH_NONE) { + if (tags->dest_error_msg == false) { + return "No Source specifier"; + } else { + return "No destination specifier"; + } + } else if (dest != 1) { + if (tags->dest_error_msg == false) { + return "Multiple Sources Specified"; + } else { + return "Multiple Destinations Specified"; + } + } + p->type = type; + p->dest_entry = 0; + p->num_el = 0; + p->line[0][0] = const_cast(label); + p->line[1][0] = const_cast(val.data()); + p->next = nullptr; + return nullptr; +} From 21901894ebccad29e6de4e15ef2e6dc0ac0022e3 Mon Sep 17 00:00:00 2001 From: John Rushford Date: Fri, 31 May 2019 21:07:18 +0000 Subject: [PATCH 2/2] fix the parent schema per review comments. --- configs/parent.schema.json | 15 +++------------ 1 file changed, 3 insertions(+), 12 deletions(-) diff --git a/configs/parent.schema.json b/configs/parent.schema.json index 1416da4908b..80448bf4e89 100644 --- a/configs/parent.schema.json +++ b/configs/parent.schema.json @@ -27,17 +27,11 @@ }, "internal": { "description": "A boolean value, true or false, specifying if the rule should match (or not match) a transaction originating from an internal API.", - "enum": [ - true, - false - ] + "type": "boolean" }, "go_direct": { "description": "send the request directly to the origin bypassing the parent list.", - "enum": [ - true, - false - ] + "type": "boolean" }, "max_simple_retries": { "description": "The maximum number of retries for a simple retry.", @@ -72,10 +66,7 @@ }, "parent_is_proxy": { "description": "The parents listed in parent and secondary_parent are caching proxies, set to false if they are origins", - "enum": [ - true, - false - ] + "type": "boolean" }, "port": { "description": "A requested URL port.",