From 16ca8784f231ea1f38221df9bc9eafebf4bfedb5 Mon Sep 17 00:00:00 2001 From: Randall Meyer Date: Wed, 17 Oct 2018 10:27:34 -0600 Subject: [PATCH] Adds voluspa - a configuration generator This generates configuration files for ATS from simpler high-level YAML specifications. It is a stand-alone command line utility and package implemented in Go. --- ci/rat-regex.txt | 5 + tools/voluspa/.gitignore | 4 + tools/voluspa/.gometalint.conf | 5 + tools/voluspa/Godeps/Godeps.json | 36 + tools/voluspa/Godeps/Readme | 5 + tools/voluspa/LICENSE | 202 ++ tools/voluspa/Makefile | 147 + tools/voluspa/README.md | 47 + tools/voluspa/adapter_config.go | 282 ++ tools/voluspa/adapters.go | 115 + tools/voluspa/adapters/access_approval.go | 83 + tools/voluspa/adapters/allow_ip.go | 86 + tools/voluspa/adapters/allow_ip_test.go | 85 + tools/voluspa/adapters/allow_methods.go | 65 + tools/voluspa/adapters/auth_proxy.go | 67 + tools/voluspa/adapters/cache_promote.go | 134 + tools/voluspa/adapters/cache_promote_test.go | 107 + tools/voluspa/adapters/cachekey.go | 125 + .../adapters/confremap/base_conf_remap.go | 63 + .../adapters/confremap/cache_version.go | 60 + tools/voluspa/adapters/confremap/common.go | 24 + .../voluspa/adapters/confremap/conf_remap.go | 107 + .../adapters/confremap/disable_cache.go | 51 + .../adapters/confremap/negative_caching.go | 68 + .../confremap/preserve_host_header.go | 82 + .../adapters/confremap/transaction_timeout.go | 68 + tools/voluspa/adapters/deny_methods.go | 69 + tools/voluspa/adapters/escalate.go | 150 + tools/voluspa/adapters/escalate_test.go | 86 + tools/voluspa/adapters/gzip.go | 198 ++ .../adapters/headerrewrite/add_header.go | 75 + .../headerrewrite/base_header_rewrite.go | 65 + .../voluspa/adapters/headerrewrite/common.go | 24 + .../headerrewrite/content_type_forge.go | 88 + .../adapters/headerrewrite/defaultttl.go | 83 + .../adapters/headerrewrite/defaultttl_test.go | 97 + tools/voluspa/adapters/headerrewrite/dscp.go | 92 + .../adapters/headerrewrite/forcettl.go | 82 + .../adapters/headerrewrite/forcettl_test.go | 95 + .../adapters/headerrewrite/header_rewrite.go | 77 + .../headerrewrite/log_cookie_header.go | 107 + .../adapters/headerrewrite/log_type.go | 74 + .../headerrewrite/proxy_cache_control.go | 77 + .../voluspa/adapters/headerrewrite/receipt.go | 101 + .../adapters/headerrewrite/remove_header.go | 74 + .../adapters/headerrewrite/rewrite_utils.go | 23 + .../adapters/headerrewrite/set_header.go | 75 + tools/voluspa/adapters/init.go | 24 + tools/voluspa/adapters/lua/base_lua.go | 65 + tools/voluspa/adapters/lua/common.go | 25 + tools/voluspa/adapters/lua/echo_cors.go | 83 + tools/voluspa/adapters/lua/lua.go | 78 + tools/voluspa/adapters/null.go | 51 + tools/voluspa/adapters/redirect.go | 87 + tools/voluspa/adapters/regex_remap.go | 76 + tools/voluspa/adapters/s3_auth.go | 139 + tools/voluspa/adapters/strip_query.go | 57 + tools/voluspa/adapters/util/ttl_utils.go | 62 + tools/voluspa/adapters/util/ttl_utils_test.go | 85 + tools/voluspa/adapters/util/util.go | 46 + .../adapters/video_background_fetch.go | 198 ++ .../adapters/video_background_fetch_test.go | 103 + tools/voluspa/attributes.go | 56 + tools/voluspa/cdncheck.go | 81 + tools/voluspa/cmd/voluspa/main.go | 172 ++ tools/voluspa/cmd/voluspa/main_test.go | 33 + tools/voluspa/cmd/voluspa/no_test.go | 20 + tools/voluspa/config_generator.go | 567 ++++ tools/voluspa/config_writer.go | 186 ++ tools/voluspa/consts.go | 30 + tools/voluspa/customer_config.go | 197 ++ tools/voluspa/doc.go | 100 + tools/voluspa/haproxy.go | 81 + tools/voluspa/hosting_config.go | 213 ++ tools/voluspa/internal/util/regex/regex.go | 24 + .../voluspa/internal/util/regex/regex_test.go | 45 + tools/voluspa/internal/util/regex/vregex.go | 33 + tools/voluspa/lifecycle.go | 29 + tools/voluspa/managed_file.go | 51 + tools/voluspa/options.go | 30 + tools/voluspa/parent_config.go | 393 +++ tools/voluspa/parent_config_test.go | 47 + tools/voluspa/property_config.go | 326 ++ tools/voluspa/property_remaps.go | 168 + tools/voluspa/receiptsd.go | 32 + tools/voluspa/registry.go | 132 + tools/voluspa/remap_options.go | 173 ++ tools/voluspa/remap_options_test.go | 66 + tools/voluspa/sample.conf-template | 178 ++ tools/voluspa/schema_v1.json | 1126 +++++++ tools/voluspa/specs/voluspa.spec | 53 + tools/voluspa/ssl_multicert.go | 199 ++ tools/voluspa/tests/.gitignore | 5 + tools/voluspa/tests/Makefile | 53 + tools/voluspa/tests/README.md | 16 + tools/voluspa/tests/other/all_bad/.gitignore | 1 + .../tests/other/all_bad/base/stderr.txt | 14 + .../all_bad/duplicated_property_name.conf | 13 + tools/voluspa/tests/other/all_bad/simple.conf | 13 + .../voluspa/tests/other/all_bad/simple2.conf | 25 + .../tests/other/cachekey_bad/base/stderr.txt | 3 + .../tests/other/cachekey_bad/test.conf | 28 + .../voluspa/tests/other/duped/base/stderr.txt | 2 + tools/voluspa/tests/other/duped/test.conf | 27 + tools/voluspa/tests/other/properties.sh | 19 + tools/voluspa/tests/other/regen_baseline.sh | 37 + .../tests/other/scheme/base/stderr.txt | 2 + tools/voluspa/tests/other/scheme/test.conf | 9 + tools/voluspa/tests/other/test.sh | 47 + .../tests/other/unicode/base/stderr.txt | 1 + .../voluspa/tests/other/unicode/httpbin.conf | 16 + .../tests/regress/all/base/all/all.config | 196 ++ .../tests/regress/all/base/all/gzip2.config | 10 + .../tests/regress/all/base/all/hdrs10.config | 4 + .../tests/regress/all/base/all/hdrs2.config | 52 + .../tests/regress/all/base/all/hdrs6.config | 4 + .../regress/all/base/all/redirect9.config | 3 + .../tests/regress/all/base/all/regex2.config | 3 + .../regress/all/base/all/strip_query2.config | 3 + .../regress/all/base/hosting.config_default | 7 + .../all/base/ssl_multicert.config_default | 9 + tools/voluspa/tests/regress/all/test.conf | 239 ++ tools/voluspa/tests/regress/properties.sh | 18 + tools/voluspa/tests/regress/regen_baseline.sh | 36 + tools/voluspa/tests/regress/test.sh | 52 + tools/voluspa/toplevel_remaps.go | 197 ++ tools/voluspa/util.go | 32 + tools/voluspa/validate.go | 312 ++ tools/voluspa/vendor/.gitignore | 9 + .../gojsonpointer/LICENSE-APACHE-2.0.txt | 202 ++ .../xeipuuv/gojsonpointer/pointer.go | 211 ++ .../gojsonreference/LICENSE-APACHE-2.0.txt | 202 ++ .../xeipuuv/gojsonreference/reference.go | 147 + .../xeipuuv/gojsonschema/.gitignore | 3 + .../gojsonschema/LICENSE-APACHE-2.0.txt | 202 ++ .../github.com/xeipuuv/gojsonschema/errors.go | 324 ++ .../xeipuuv/gojsonschema/format_checkers.go | 343 +++ .../xeipuuv/gojsonschema/internalLog.go | 37 + .../xeipuuv/gojsonschema/jsonContext.go | 72 + .../xeipuuv/gojsonschema/jsonLoader.go | 362 +++ .../xeipuuv/gojsonschema/locales.go | 313 ++ .../github.com/xeipuuv/gojsonschema/result.go | 195 ++ .../github.com/xeipuuv/gojsonschema/schema.go | 1000 ++++++ .../xeipuuv/gojsonschema/schemaLoader.go | 102 + .../xeipuuv/gojsonschema/schemaPool.go | 201 ++ .../gojsonschema/schemaReferencePool.go | 68 + .../xeipuuv/gojsonschema/schemaType.go | 83 + .../xeipuuv/gojsonschema/subSchema.go | 255 ++ .../github.com/xeipuuv/gojsonschema/types.go | 58 + .../github.com/xeipuuv/gojsonschema/utils.go | 226 ++ .../xeipuuv/gojsonschema/validation.go | 928 ++++++ tools/voluspa/vendor/gopkg.in/yaml.v2/LICENSE | 201 ++ .../vendor/gopkg.in/yaml.v2/LICENSE.libyaml | 31 + tools/voluspa/vendor/gopkg.in/yaml.v2/apic.go | 739 +++++ .../voluspa/vendor/gopkg.in/yaml.v2/decode.go | 775 +++++ .../vendor/gopkg.in/yaml.v2/emitterc.go | 1685 +++++++++++ .../voluspa/vendor/gopkg.in/yaml.v2/encode.go | 390 +++ .../vendor/gopkg.in/yaml.v2/parserc.go | 1095 +++++++ .../vendor/gopkg.in/yaml.v2/readerc.go | 412 +++ .../vendor/gopkg.in/yaml.v2/resolve.go | 258 ++ .../vendor/gopkg.in/yaml.v2/scannerc.go | 2696 +++++++++++++++++ .../voluspa/vendor/gopkg.in/yaml.v2/sorter.go | 113 + .../vendor/gopkg.in/yaml.v2/writerc.go | 26 + tools/voluspa/vendor/gopkg.in/yaml.v2/yaml.go | 466 +++ .../voluspa/vendor/gopkg.in/yaml.v2/yamlh.go | 738 +++++ .../vendor/gopkg.in/yaml.v2/yamlprivateh.go | 173 ++ tools/voluspa/version.go | 28 + tools/voluspa/version/version.go | 39 + tools/voluspa/version/version_test.go | 20 + tools/voluspa/voluspa.go | 330 ++ 170 files changed, 27221 insertions(+) create mode 100644 tools/voluspa/.gitignore create mode 100644 tools/voluspa/.gometalint.conf create mode 100644 tools/voluspa/Godeps/Godeps.json create mode 100644 tools/voluspa/Godeps/Readme create mode 100644 tools/voluspa/LICENSE create mode 100644 tools/voluspa/Makefile create mode 100644 tools/voluspa/README.md create mode 100644 tools/voluspa/adapter_config.go create mode 100644 tools/voluspa/adapters.go create mode 100644 tools/voluspa/adapters/access_approval.go create mode 100644 tools/voluspa/adapters/allow_ip.go create mode 100644 tools/voluspa/adapters/allow_ip_test.go create mode 100644 tools/voluspa/adapters/allow_methods.go create mode 100644 tools/voluspa/adapters/auth_proxy.go create mode 100644 tools/voluspa/adapters/cache_promote.go create mode 100644 tools/voluspa/adapters/cache_promote_test.go create mode 100644 tools/voluspa/adapters/cachekey.go create mode 100644 tools/voluspa/adapters/confremap/base_conf_remap.go create mode 100644 tools/voluspa/adapters/confremap/cache_version.go create mode 100644 tools/voluspa/adapters/confremap/common.go create mode 100644 tools/voluspa/adapters/confremap/conf_remap.go create mode 100644 tools/voluspa/adapters/confremap/disable_cache.go create mode 100644 tools/voluspa/adapters/confremap/negative_caching.go create mode 100644 tools/voluspa/adapters/confremap/preserve_host_header.go create mode 100644 tools/voluspa/adapters/confremap/transaction_timeout.go create mode 100644 tools/voluspa/adapters/deny_methods.go create mode 100644 tools/voluspa/adapters/escalate.go create mode 100644 tools/voluspa/adapters/escalate_test.go create mode 100644 tools/voluspa/adapters/gzip.go create mode 100644 tools/voluspa/adapters/headerrewrite/add_header.go create mode 100644 tools/voluspa/adapters/headerrewrite/base_header_rewrite.go create mode 100644 tools/voluspa/adapters/headerrewrite/common.go create mode 100644 tools/voluspa/adapters/headerrewrite/content_type_forge.go create mode 100644 tools/voluspa/adapters/headerrewrite/defaultttl.go create mode 100644 tools/voluspa/adapters/headerrewrite/defaultttl_test.go create mode 100644 tools/voluspa/adapters/headerrewrite/dscp.go create mode 100644 tools/voluspa/adapters/headerrewrite/forcettl.go create mode 100644 tools/voluspa/adapters/headerrewrite/forcettl_test.go create mode 100644 tools/voluspa/adapters/headerrewrite/header_rewrite.go create mode 100644 tools/voluspa/adapters/headerrewrite/log_cookie_header.go create mode 100644 tools/voluspa/adapters/headerrewrite/log_type.go create mode 100644 tools/voluspa/adapters/headerrewrite/proxy_cache_control.go create mode 100644 tools/voluspa/adapters/headerrewrite/receipt.go create mode 100644 tools/voluspa/adapters/headerrewrite/remove_header.go create mode 100644 tools/voluspa/adapters/headerrewrite/rewrite_utils.go create mode 100644 tools/voluspa/adapters/headerrewrite/set_header.go create mode 100644 tools/voluspa/adapters/init.go create mode 100644 tools/voluspa/adapters/lua/base_lua.go create mode 100644 tools/voluspa/adapters/lua/common.go create mode 100644 tools/voluspa/adapters/lua/echo_cors.go create mode 100644 tools/voluspa/adapters/lua/lua.go create mode 100644 tools/voluspa/adapters/null.go create mode 100644 tools/voluspa/adapters/redirect.go create mode 100644 tools/voluspa/adapters/regex_remap.go create mode 100644 tools/voluspa/adapters/s3_auth.go create mode 100644 tools/voluspa/adapters/strip_query.go create mode 100644 tools/voluspa/adapters/util/ttl_utils.go create mode 100644 tools/voluspa/adapters/util/ttl_utils_test.go create mode 100644 tools/voluspa/adapters/util/util.go create mode 100644 tools/voluspa/adapters/video_background_fetch.go create mode 100644 tools/voluspa/adapters/video_background_fetch_test.go create mode 100644 tools/voluspa/attributes.go create mode 100644 tools/voluspa/cdncheck.go create mode 100644 tools/voluspa/cmd/voluspa/main.go create mode 100644 tools/voluspa/cmd/voluspa/main_test.go create mode 100644 tools/voluspa/cmd/voluspa/no_test.go create mode 100644 tools/voluspa/config_generator.go create mode 100644 tools/voluspa/config_writer.go create mode 100644 tools/voluspa/consts.go create mode 100644 tools/voluspa/customer_config.go create mode 100644 tools/voluspa/doc.go create mode 100644 tools/voluspa/haproxy.go create mode 100644 tools/voluspa/hosting_config.go create mode 100644 tools/voluspa/internal/util/regex/regex.go create mode 100644 tools/voluspa/internal/util/regex/regex_test.go create mode 100644 tools/voluspa/internal/util/regex/vregex.go create mode 100644 tools/voluspa/lifecycle.go create mode 100644 tools/voluspa/managed_file.go create mode 100644 tools/voluspa/options.go create mode 100644 tools/voluspa/parent_config.go create mode 100644 tools/voluspa/parent_config_test.go create mode 100644 tools/voluspa/property_config.go create mode 100644 tools/voluspa/property_remaps.go create mode 100644 tools/voluspa/receiptsd.go create mode 100644 tools/voluspa/registry.go create mode 100644 tools/voluspa/remap_options.go create mode 100644 tools/voluspa/remap_options_test.go create mode 100644 tools/voluspa/sample.conf-template create mode 100644 tools/voluspa/schema_v1.json create mode 100644 tools/voluspa/specs/voluspa.spec create mode 100644 tools/voluspa/ssl_multicert.go create mode 100644 tools/voluspa/tests/.gitignore create mode 100644 tools/voluspa/tests/Makefile create mode 100644 tools/voluspa/tests/README.md create mode 100644 tools/voluspa/tests/other/all_bad/.gitignore create mode 100644 tools/voluspa/tests/other/all_bad/base/stderr.txt create mode 100644 tools/voluspa/tests/other/all_bad/duplicated_property_name.conf create mode 100644 tools/voluspa/tests/other/all_bad/simple.conf create mode 100644 tools/voluspa/tests/other/all_bad/simple2.conf create mode 100644 tools/voluspa/tests/other/cachekey_bad/base/stderr.txt create mode 100644 tools/voluspa/tests/other/cachekey_bad/test.conf create mode 100644 tools/voluspa/tests/other/duped/base/stderr.txt create mode 100644 tools/voluspa/tests/other/duped/test.conf create mode 100644 tools/voluspa/tests/other/properties.sh create mode 100755 tools/voluspa/tests/other/regen_baseline.sh create mode 100644 tools/voluspa/tests/other/scheme/base/stderr.txt create mode 100644 tools/voluspa/tests/other/scheme/test.conf create mode 100755 tools/voluspa/tests/other/test.sh create mode 100644 tools/voluspa/tests/other/unicode/base/stderr.txt create mode 100644 tools/voluspa/tests/other/unicode/httpbin.conf create mode 100644 tools/voluspa/tests/regress/all/base/all/all.config create mode 100644 tools/voluspa/tests/regress/all/base/all/gzip2.config create mode 100644 tools/voluspa/tests/regress/all/base/all/hdrs10.config create mode 100644 tools/voluspa/tests/regress/all/base/all/hdrs2.config create mode 100644 tools/voluspa/tests/regress/all/base/all/hdrs6.config create mode 100644 tools/voluspa/tests/regress/all/base/all/redirect9.config create mode 100644 tools/voluspa/tests/regress/all/base/all/regex2.config create mode 100644 tools/voluspa/tests/regress/all/base/all/strip_query2.config create mode 100644 tools/voluspa/tests/regress/all/base/hosting.config_default create mode 100644 tools/voluspa/tests/regress/all/base/ssl_multicert.config_default create mode 100644 tools/voluspa/tests/regress/all/test.conf create mode 100644 tools/voluspa/tests/regress/properties.sh create mode 100755 tools/voluspa/tests/regress/regen_baseline.sh create mode 100755 tools/voluspa/tests/regress/test.sh create mode 100644 tools/voluspa/toplevel_remaps.go create mode 100644 tools/voluspa/util.go create mode 100644 tools/voluspa/validate.go create mode 100644 tools/voluspa/vendor/.gitignore create mode 100644 tools/voluspa/vendor/github.com/xeipuuv/gojsonpointer/LICENSE-APACHE-2.0.txt create mode 100644 tools/voluspa/vendor/github.com/xeipuuv/gojsonpointer/pointer.go create mode 100644 tools/voluspa/vendor/github.com/xeipuuv/gojsonreference/LICENSE-APACHE-2.0.txt create mode 100644 tools/voluspa/vendor/github.com/xeipuuv/gojsonreference/reference.go create mode 100644 tools/voluspa/vendor/github.com/xeipuuv/gojsonschema/.gitignore create mode 100644 tools/voluspa/vendor/github.com/xeipuuv/gojsonschema/LICENSE-APACHE-2.0.txt create mode 100644 tools/voluspa/vendor/github.com/xeipuuv/gojsonschema/errors.go create mode 100644 tools/voluspa/vendor/github.com/xeipuuv/gojsonschema/format_checkers.go create mode 100644 tools/voluspa/vendor/github.com/xeipuuv/gojsonschema/internalLog.go create mode 100644 tools/voluspa/vendor/github.com/xeipuuv/gojsonschema/jsonContext.go create mode 100644 tools/voluspa/vendor/github.com/xeipuuv/gojsonschema/jsonLoader.go create mode 100644 tools/voluspa/vendor/github.com/xeipuuv/gojsonschema/locales.go create mode 100644 tools/voluspa/vendor/github.com/xeipuuv/gojsonschema/result.go create mode 100644 tools/voluspa/vendor/github.com/xeipuuv/gojsonschema/schema.go create mode 100644 tools/voluspa/vendor/github.com/xeipuuv/gojsonschema/schemaLoader.go create mode 100644 tools/voluspa/vendor/github.com/xeipuuv/gojsonschema/schemaPool.go create mode 100644 tools/voluspa/vendor/github.com/xeipuuv/gojsonschema/schemaReferencePool.go create mode 100644 tools/voluspa/vendor/github.com/xeipuuv/gojsonschema/schemaType.go create mode 100644 tools/voluspa/vendor/github.com/xeipuuv/gojsonschema/subSchema.go create mode 100644 tools/voluspa/vendor/github.com/xeipuuv/gojsonschema/types.go create mode 100644 tools/voluspa/vendor/github.com/xeipuuv/gojsonschema/utils.go create mode 100644 tools/voluspa/vendor/github.com/xeipuuv/gojsonschema/validation.go create mode 100644 tools/voluspa/vendor/gopkg.in/yaml.v2/LICENSE create mode 100644 tools/voluspa/vendor/gopkg.in/yaml.v2/LICENSE.libyaml create mode 100644 tools/voluspa/vendor/gopkg.in/yaml.v2/apic.go create mode 100644 tools/voluspa/vendor/gopkg.in/yaml.v2/decode.go create mode 100644 tools/voluspa/vendor/gopkg.in/yaml.v2/emitterc.go create mode 100644 tools/voluspa/vendor/gopkg.in/yaml.v2/encode.go create mode 100644 tools/voluspa/vendor/gopkg.in/yaml.v2/parserc.go create mode 100644 tools/voluspa/vendor/gopkg.in/yaml.v2/readerc.go create mode 100644 tools/voluspa/vendor/gopkg.in/yaml.v2/resolve.go create mode 100644 tools/voluspa/vendor/gopkg.in/yaml.v2/scannerc.go create mode 100644 tools/voluspa/vendor/gopkg.in/yaml.v2/sorter.go create mode 100644 tools/voluspa/vendor/gopkg.in/yaml.v2/writerc.go create mode 100644 tools/voluspa/vendor/gopkg.in/yaml.v2/yaml.go create mode 100644 tools/voluspa/vendor/gopkg.in/yaml.v2/yamlh.go create mode 100644 tools/voluspa/vendor/gopkg.in/yaml.v2/yamlprivateh.go create mode 100644 tools/voluspa/version.go create mode 100644 tools/voluspa/version/version.go create mode 100644 tools/voluspa/version/version_test.go create mode 100644 tools/voluspa/voluspa.go diff --git a/ci/rat-regex.txt b/ci/rat-regex.txt index 18c5c827e59..a7aacf8029b 100644 --- a/ci/rat-regex.txt +++ b/ci/rat-regex.txt @@ -67,3 +67,8 @@ port\.h ^catch[.]hpp$ ^configuru.hpp$ ^yamlcpp$ +# tools/voluspa related +^.gometalint.conf$ +.*\.conf$ +.*config_default$ +gojsonschema(?: Apache 2.0. Properly documented in LICENSE-APACHE-2.0.txt ){0} diff --git a/tools/voluspa/.gitignore b/tools/voluspa/.gitignore new file mode 100644 index 00000000000..09a98f8e11d --- /dev/null +++ b/tools/voluspa/.gitignore @@ -0,0 +1,4 @@ +voluspa +voluspa-covered +/dist +/RPMS diff --git a/tools/voluspa/.gometalint.conf b/tools/voluspa/.gometalint.conf new file mode 100644 index 00000000000..c3719752a3d --- /dev/null +++ b/tools/voluspa/.gometalint.conf @@ -0,0 +1,5 @@ +{ + "$comment": "Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements; and to You under the Apache License, Version 2.0.", + "EnableAll": true, + "Disable": ["gosec", "gas", "golint", "vetshadow", "gocyclo"] +} diff --git a/tools/voluspa/Godeps/Godeps.json b/tools/voluspa/Godeps/Godeps.json new file mode 100644 index 00000000000..3ef1bdf7daf --- /dev/null +++ b/tools/voluspa/Godeps/Godeps.json @@ -0,0 +1,36 @@ +{ + "ImportPath": "github.com/apache/trafficserver/tools/voluspa", + "GoVersion": "go1.12", + "GodepVersion": "v80", + "Packages": [ + ".", + "./adapters", + "./adapters/confremap", + "./adapters/headerrewrite", + "./adapters/lua", + "./adapters/util", + "./cmd/voluspa", + "./internal/util/regex", + "./version" + ], + "Deps": [ + { + "ImportPath": "github.com/xeipuuv/gojsonpointer", + "Rev": "4e3ac2762d5f479393488629ee9370b50873b3a6" + }, + { + "ImportPath": "github.com/xeipuuv/gojsonreference", + "Rev": "bd5ef7bd5415a7ac448318e64f11a24cd21e594b" + }, + { + "ImportPath": "github.com/xeipuuv/gojsonschema", + "Comment": "v1.1.0-14-g354ad34", + "Rev": "354ad34c2300dd6e84920ac70b41487336219e43" + }, + { + "ImportPath": "gopkg.in/yaml.v2", + "Comment": "v2.2.2-1-g7b8349a", + "Rev": "7b8349ac747c6a24702b762d2c4fd9266cf4f1d6" + } + ] +} diff --git a/tools/voluspa/Godeps/Readme b/tools/voluspa/Godeps/Readme new file mode 100644 index 00000000000..4cdaa53d56d --- /dev/null +++ b/tools/voluspa/Godeps/Readme @@ -0,0 +1,5 @@ +This directory tree is generated automatically by godep. + +Please do not edit. + +See https://github.com/tools/godep for more information. diff --git a/tools/voluspa/LICENSE b/tools/voluspa/LICENSE new file mode 100644 index 00000000000..d6456956733 --- /dev/null +++ b/tools/voluspa/LICENSE @@ -0,0 +1,202 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/tools/voluspa/Makefile b/tools/voluspa/Makefile new file mode 100644 index 00000000000..c75fdd37967 --- /dev/null +++ b/tools/voluspa/Makefile @@ -0,0 +1,147 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +APP := voluspa +RELEASE ?= 0 +VERSION_BASE := 24A +BUILD_NUMBER ?= devel +VERSION := $(VERSION_BASE)$(BUILD_NUMBER) +GIT_VERSION := $(shell git describe --always --long) +PKG := github.com/apache/trafficserver/tools/voluspa +LDFLAGS := -ldflags "-w -X $(PKG).Version=$(VERSION) -X $(PKG).GitVersion=$(GIT_VERSION)" +DIST := dist +ARCHIVE := $(APP)-$(VERSION) +CURRENT_SCHEMA := schema_v1.json +DEST ?= . +GO ?= go + +COMMON_GO_FILES := *.go */*.go internal/util/*/*go */*/*.go + +GO_PKG := $(shell go list ./... | egrep -v 'vendor|cmd|other') +GO_PKG_DIRS := $(subst $(shell go list -e .),.,$(shell go list ./... | egrep -v 'vendor|other')) + +CMD_SOURCES := $(shell find cmd -name main.go) +DEV_TARGETS := $(patsubst cmd/%/main.go,%,$(CMD_SOURCES)) +DIST_TARGETS := voluspa +LINUX_TARGETS := $(patsubst %,dist/linux-x86_64/%,$(DIST_TARGETS)) +DARWIN_TARGETS := $(patsubst %,dist/darwin-x86_64/%,$(DIST_TARGETS)) + +all: default + +generate: + go generate ./... + +default: $(DEV_TARGETS) + +sanity_tests: $(APP) + @make -C tests test + +test: $(APP) sanity_tests + $(GO) test -cover $(GO_PKG_DIRS) + +lint: + -$(GO) vet ./... 2>&1 + -golint $(GO_PKG) | egrep -v 'vendor|unexport' + +metalint: + -gometalinter --config .gometalint.conf --deadline=60s -e 'vendor' ./... + +junit: + @mkdir -p .out/test-results/ # where rio junit support looks by default + $(GO) test -v $(GO_PKG_DIRS) 2>&1 | go-junit-report > .out/test-results/junit-report.xml + +cover: + # go get github.com/wadey/gocovmerge github.com/axw/gocov/... github.com/matm/gocov-html + # based on https://github.com/golang/go/issues/6909#issuecomment-233493644 + @echo Generating an overall coverage report + @rm -f *.coverprofile + go list -f '{{if or (len .TestGoFiles) (len .XTestGoFiles)}}go test -test.timeout=120s -covermode=count -coverprofile={{.Name}}_{{len .Imports}}_{{len .Deps}}.coverprofile -coverpkg ./... {{.ImportPath}}{{end}}' $(GO_PKG) | sh -x + gocovmerge `ls *.coverprofile` > cover.out + @rm -f *.coverprofile + @mkdir -p .out/coverage/ + gocov convert cover.out | gocov-html > .out/coverage/merged-coverage.html + @[[ $$(uname) == 'Darwin' ]] && open .out/coverage/merged-coverage.html || true + +voluspa-covered: cmd/voluspa/*.go $(COMMON_GO_FILES) + @# build a test version of the voluspa command with coverage enabled + @# e.g.: ./voluspa-covered -test.coverprofile cover.out .../*.conf + go test -o $(DEST)/$@ -c -tags testrunmain -coverpkg ./... $(PKG)/cmd/voluspa + +gocov: .out/coverage/coverage.html + +.out/coverage/coverage.json: voluspa-covered + @mkdir -p .out/coverage/ + gocov test $(GO_PKG) > .out/coverage/coverage.json + @make -C tests coverage + gocov convert tests/regress/profile*.out > tests/regress/system.json + gocov convert tests/other/profile*.out > tests/other/system.json + cat .out/coverage/coverage.json tests/other/system.json tests/regress/system.json | gocov-merge > .out/coverage/coverage.json + +.out/coverage/coverage.html: .out/coverage/coverage.json + gocov-html < .out/coverage/coverage.json > .out/coverage/coverage.html + +dist: artifactory_targets $(DIST)/artifactory/version.txt $(DIST)/artifactory/versions.sh + +clean: + @rm -rf $(DIST) out out_remap RPMS .dist .out + @rm -f $(DEV_TARGETS) $(TOOLS_TARGETS) $(OTHER_TARGETS) voluspa-covered + @make -C tests clean + +%: cmd/%/*.go $(COMMON_GO_FILES) + $(GO) build -o $(DEST)/$@ $(LDFLAGS) cmd/$@/*.go + +$(DIST)/linux-x86_64/%: + GOOS=linux GOARCH=amd64 $(GO) build -o $@ $(LDFLAGS) cmd/$(notdir $@)/*.go + +$(DIST)/darwin-x86_64/%: + GOOS=darwin GOARCH=amd64 $(GO) build -o $@ $(LDFLAGS) cmd/$(notdir $@)/*.go + +$(DIST)/freebsd-x86_64/%: + GOOS=freebsd GOARCH=amd64 $(GO) build -o $@ $(LDFLAGS) cmd/$(notdir $@)/*.go + +$(DIST)/artifactory: + @mkdir -p $@ + +$(DIST)/artifactory/$(APP)-$(VERSION)-darwin-x86_64: $(DIST)/artifactory dist/darwin-x86_64/voluspa + cp dist/darwin-x86_64/voluspa $@ + +$(DIST)/artifactory/$(APP)-$(VERSION)-linux-x86_64: $(DIST)/artifactory dist/linux-x86_64/voluspa + cp dist/linux-x86_64/voluspa $@ + +$(DIST)/artifactory/$(CURRENT_SCHEMA)-$(VERSION): $(DIST)/artifactory $(CURRENT_SCHEMA) + cp $(CURRENT_SCHEMA) $@ + +$(DIST)/artifactory/version.txt: + @mkdir -p dist/artifactory + echo $(VERSION) > $@ + +artifactory_targets: $(DIST)/artifactory/$(APP)-$(VERSION)-darwin-x86_64 $(DIST)/artifactory/$(APP)-$(VERSION)-linux-x86_64 $(DIST)/artifactory/$(CURRENT_SCHEMA)-$(VERSION) + +$(DIST)/artifactory/versions.sh: $(DIST)/artifactory/$(APP)-$(VERSION)-darwin-x86_64 $(DIST)/artifactory/$(APP)-$(VERSION)-linux-x86_64 + echo VOLUSPA_VERSION=$(VERSION) > $@ + echo >> $@ + sha1sum $(DIST)/artifactory/$(APP)-$(VERSION)-linux-x86_64 | awk 'BEGIN { FS = " " } ; { printf "linux_SHA1=%s\n", $$1 }' >> $@ + sha1sum $(DIST)/artifactory/$(APP)-$(VERSION)-darwin-x86_64 | awk 'BEGIN { FS = " " } ; { printf "darwin_SHA1=%s\n", $$1 }' >> $@ + sha1sum $(DIST)/artifactory/$(CURRENT_SCHEMA)-$(VERSION) | awk 'BEGIN { FS = " " } ; { printf "schema_SHA1=%s\n", $$1 }' >> $@ + +update-deps: + -grep ImportPath Godeps/Godeps.json | cut -f 4 -d \" | grep -v voluspa | xargs go get -u + @rm -rf Godeps vendor/* + godep save $(GO_PKG_DIRS) + +.PHONY: all clean test linux dist sanity_tests diff --git a/tools/voluspa/README.md b/tools/voluspa/README.md new file mode 100644 index 00000000000..f712ccc34bb --- /dev/null +++ b/tools/voluspa/README.md @@ -0,0 +1,47 @@ +# Voluspa + +### Apache Traffic Server Configuration Tool (experiment) + +A tool to codify the CDN feature set and make it easier and more reliable to setup new CDN properties. + +Configuration files are well-formed YAML files that express the desired behaviour. + + +## Usage +``` +$ ./voluspa -dest=ourconfigs myproperty.conf + +$ find ourconfigs -type f +ourconfigs/ssl_multicert.config_tenant +ourconfigs/myproperty/myproperty.config +ourconfigs/myproperty/hdrs.config +ourconfigs/ssl_multicert.config_default +ourconfigs/voluspa.sls +``` + +The voluspa configuration language is defined by a +[JSON Schema](https://spacetelescope.github.io/understanding-json-schema/index.html) +stored in the [schema file](schema_v1.json). + +Tool will generally error out if something is wrong (missing required fields) or weirdly configured options. +Warnings will be generated for unknown configuration options (but will attempt to still process the file). +There are a number of options to control the output and output location (see -help) + +By default, all configs get a receipt header and propstats enabled unless disabled via the appropriate option. + +## Extending + +### General process includes: +- adding the shared library name (sharedLibraryNames) +- configuration option name (see pluginConfigNameMap) +- weighting the plugin (for sorting) (pluginWeights) +- most importantly, the class/struct/file for the plugin +- a bit more complicated if it's a "compound plugin" (eg HeaderRewrite) + + +## Tests +There are a couple of go-based unit tests (no where near comprehensive) and there's what I would call sanity tests in tests/. The script "test.sh" will use the current build of the tool to regenerate the remap configs for a given yaml file; and do a diff between base/ and the new output. Generally, there should be no changes between runs. If there are expected changes, the baselines would need to be regenerated. The configurations tested range from very simple to something exercising all current configuration options. + + +## TODO +- error on mutually exclusive options (eg allow\_ip and deny\_methods) diff --git a/tools/voluspa/adapter_config.go b/tools/voluspa/adapter_config.go new file mode 100644 index 00000000000..1ee9c03b4c1 --- /dev/null +++ b/tools/voluspa/adapter_config.go @@ -0,0 +1,282 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package voluspa + +import ( + "bytes" + "fmt" + "sort" + "strings" +) + +type AdapterType string + +const ( + UnknownAdapter AdapterType = "unknown" + Receipt = "receipt" +) + +type ConfigLocation int + +const ( + UnknownLocation ConfigLocation = iota + ChildConfig + ParentConfig +) + +// validConfigrationOptions are options that are not plugin specific (which are managed by the plugin) +var validConfigurationOptions = map[string]bool{ + "parent_child": true, + Parent: true, + Child: true, +} + +type mappingRule struct { + Type AdapterType + Weight int + SubConfigFileName string + SubConfigContent *bytes.Buffer + ConfigContent *bytes.Buffer +} + +func newMappingRule(vp Adapter) *mappingRule { + return &mappingRule{Type: vp.Type(), Weight: getWeight(vp)} +} + +// hasContent returns true if there's configuration content, false otherwise +func (p *mappingRule) hasContent() bool { + return (p.SubConfigContent != nil && p.SubConfigContent.Len() > 0) || + (p.ConfigContent != nil && p.ConfigContent.Len() > 0) +} + +func (p *mappingRule) isCommandLineTemplated() bool { + return p.ConfigContent != nil && strings.Contains(p.ConfigContent.String(), "{%") +} + +// pluginWeights affects the ordering of the configuration output. +// Only add to this list if you have hard ordering requirements, otherwise just stick +// with the default weighting (5) +func getWeight(vp Adapter) int { + t, ok := vp.(Weighty) + if !ok { + return 5 + } + return t.Weight() +} + +// SubConfigFilename returns the subconfig filename given plugin and property name +func SubConfigFilename(p SubConfigAdapter, env *ConfigEnvironment) string { + extension := "config" + name := p.Name() + if strings.HasSuffix(p.Name(), ".lua") { + extension = "lua" + name = strings.TrimSuffix(name, ".lua") + } + if env.id == 1 { + if env.ConfigLocation == ParentConfig { + return fmt.Sprintf("%s_parent.%s", name, extension) + } + return fmt.Sprintf("%s.%s", name, extension) + } + + if env.ConfigLocation == ParentConfig { + return fmt.Sprintf("%s%d_parent.%s", name, env.id, extension) + } + return fmt.Sprintf("%s%d.%s", name, env.id, extension) +} + +// ConfigFileNameSpecification appends to buffer the sub config filename configuration specification +// nolint: interfacer +func ConfigFileNameSpecification(p Adapter, env *ConfigEnvironment, buffer *bytes.Buffer) (*bytes.Buffer, error) { + var configFileSwitch string + cfs, ok := p.(ConfigFileSwitch) + if ok { + configFileSwitch = cfs.ConfigFileSwitch() + } + + scp, ok := p.(SubConfigAdapter) + if ok { + fmt.Fprintf(buffer, " @pparam=%s%s/%s", configFileSwitch, strings.ToLower(env.Property), SubConfigFilename(scp, env)) + } + return buffer, nil +} + +func isParameterBasedAdapter(p Adapter) bool { + _, ok := p.(ParameterAdapter) + if ok { + return true + } + + _, ok = p.(RoleifiedParameterAdapter) + + return ok +} + +func ConfigContent(p Adapter, env *ConfigEnvironment) (*bytes.Buffer, error) { + + var buffer bytes.Buffer + + var roleEnabled bool + rep, ok := p.(RoleEnabled) + if ok { + roleEnabled = true + fmt.Fprintf(&buffer, "{%% if salt.pillar.get(\"%s\") %%}%s", rep.Role(), remapPrefixBuffer) + } + + sla, ok := p.(SharedLibraryAdapter) + if ok { + fmt.Fprintf(&buffer, "@plugin=%s", sla.SharedLibraryName()) + } + + if !isParameterBasedAdapter(p) { + return ConfigFileNameSpecification(p, env, &buffer) + } + + var err error + var subfer *bytes.Buffer + switch vpp := p.(type) { + case ParameterAdapter: + subfer, err = processParameterAdapter(p, env, vpp) + case RoleifiedParameterAdapter: + subfer, err = processRoleifiedParameterAdapter(vpp, env) + default: + return nil, fmt.Errorf("unhandled adapter %s", vpp) + } + + if err != nil { + return nil, err + } + + // if sub-buffer is nil, pass back nil + if subfer == nil { + return nil, nil + } + + buffer.Write(subfer.Bytes()) + + if roleEnabled { + fmt.Fprintf(&buffer, " \\{%% else %%}%s\\{%% endif %%}", remapPrefixBuffer) + fmt.Fprintln(&buffer, "") + } + + return &buffer, nil +} + +func processParameterAdapter(p Adapter, env *ConfigEnvironment, vpp ParameterAdapter) (*bytes.Buffer, error) { + var buffer bytes.Buffer + + // throw away all work if we get an error (which can be a signal not to include this plugin (see propstats)) + // TODO: signal with appropriate error (like ErrSkipConfig) + pparams, err := vpp.PParams(env) + if err != nil { + return &buffer, err + } + + raw := false + rpparams, ok := vpp.(RawParameterAdapter) + if ok { + raw = rpparams.Raw() + } + + if len(pparams) == 0 { + if p.PluginType() == CompoundAdapter { + return &buffer, nil + } + // no buffer to append? + return nil, nil + } + + if raw { + for _, val := range pparams { + fmt.Fprintf(&buffer, " %s", val) + } + } else { + for _, val := range pparams { + fmt.Fprintf(&buffer, " @pparam=%s", val) + } + } + + return &buffer, nil +} + +func processRoleifiedParameterAdapter(vpp RoleifiedParameterAdapter, env *ConfigEnvironment) (*bytes.Buffer, error) { + var buffer bytes.Buffer + pparams, err := vpp.PParams(env) + if err != nil { + return &buffer, err + } + + // extract non-Default role keys from parameter list + var roleKeys []string + for key := range pparams { + if key == DefaultRole { + continue + } + roleKeys = append(roleKeys, key) + } + sort.Strings(roleKeys) + + // 2 passes + // 1. shove the pparams and role-guards into a list + // 2. iterate over that listen, building up buffer with the \n and \\ and whitespace + + var args []string + pparam, found := pparams[DefaultRole] + if found { + for _, val := range pparam { + args = append(args, fmt.Sprintf("@pparam=%s", val)) + } + } + + for _, key := range roleKeys { + args = append(args, fmt.Sprintf("{%% if salt.pillar.get(\"%s\") %%}%s", key, remapPrefixBuffer)) + + for _, val := range pparams[key] { + args = append(args, fmt.Sprintf("@pparam=%s", val)) + } + args = append(args, fmt.Sprintf(" \\{%% else %%}%s\\{%% endif %%}", remapPrefixBuffer)) + } + + // peel off last arg for special handling + lastArg := args[len(args)-1] + args = args[:len(args)-1] + + for _, arg := range args { + fmt.Fprintf(&buffer, " %s", arg) + } + + // TODO: assuming last element is {% endif %} block + fmt.Fprintf(&buffer, "%s\n", lastArg) + + return &buffer, nil +} + +func SubConfigContent(p Adapter, env *ConfigEnvironment) (*bytes.Buffer, error) { + cba, ok := p.(ContentBasedAdapter) + if ok { + return cba.Content(env) + } + return nil, nil +} + +// IsValidConfigurationOption returns true if configuration key is valid +func IsValidConfigurationOption(option string) bool { + return validConfigurationOptions[option] +} diff --git a/tools/voluspa/adapters.go b/tools/voluspa/adapters.go new file mode 100644 index 00000000000..b705011a596 --- /dev/null +++ b/tools/voluspa/adapters.go @@ -0,0 +1,115 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package voluspa + +import "bytes" + +type PluginType int + +const ( + UnknownVoluspaAdapter PluginType = iota + GeneralAdapter + ActionAdapter + CompoundAdapter +) + +type ConfigEnvironment struct { + Property string + Options Options + RemapOptions RemapOptions + ConfigLocation ConfigLocation + id int +} + +func newConfigEnvironment(p *PropertyConfig, options Options, mappingID int, configLocation ConfigLocation) *ConfigEnvironment { + return &ConfigEnvironment{ + Property: p.Property, + Options: options, + ConfigLocation: configLocation, + id: mappingID, + } +} + +// Adapter needs to be implemented by all adapters +type Adapter interface { + PluginType() PluginType + Type() AdapterType + ConfigParameters() []string +} + +// ContentBasedAdapter should be implemented if a configuration is a blob of text (opposite of ParameterAdapter) +type ContentBasedAdapter interface { + Content(*ConfigEnvironment) (*bytes.Buffer, error) +} + +// SharedLibraryAdapter should be implemented if a configuration has an associated shared library +type SharedLibraryAdapter interface { + SharedLibraryName() string +} + +// CompoundTypeAdapter should be implemented if an adapter's output is shared with other like adapters +type CompoundTypeAdapter interface { + CompoundType() AdapterType +} + +// ParameterAdapter should be implemented if a configuration has @pparams +type ParameterAdapter interface { + PParams(*ConfigEnvironment) ([]string, error) +} + +// RawParameterAdapter should be implemented if a configuration has @pparams that should not be processed +type RawParameterAdapter interface { + Raw() bool +} + +// RolePParams is a map of role names to a list of PParam options +type RolePParams map[string][]string + +// RoleifiedParameterAdapter should be implemented in adapters +// that can vary their options based on host role/type +type RoleifiedParameterAdapter interface { + PParams(*ConfigEnvironment) (RolePParams, error) +} + +// SubConfigAdapter should be implemented if a sub config is involved with rule/plugin +type SubConfigAdapter interface { + Name() string +} + +// ConfigFileSwitch should be implemented if a sub config is specified with a plugin switch (eg --config=) +type ConfigFileSwitch interface { + ConfigFileSwitch() string +} + +// ParentChildAdapter should be implemented if plugin is conditionally placed based on parent/child context +type ParentChildAdapter interface { + ConfigLocations() []ConfigLocation +} + +// RoleEnabled should be implemented if command-line should be protected/enabled by a jinja role-block +type RoleEnabled interface { + Role() string +} + +// Weighty should be implemented if an Adapter's output needs to be placed in a certain +// order (in relation to other adapter's output) +type Weighty interface { + Weight() int +} diff --git a/tools/voluspa/adapters/access_approval.go b/tools/voluspa/adapters/access_approval.go new file mode 100644 index 00000000000..b49b000ecae --- /dev/null +++ b/tools/voluspa/adapters/access_approval.go @@ -0,0 +1,83 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package adapters + +import ( + "strings" + + "github.com/apache/trafficserver/tools/voluspa" + "github.com/apache/trafficserver/tools/voluspa/adapters/util" +) + +const ( + AccessApprovalAdapterParameter = "access_approval" +) + +func init() { + voluspa.AdaptersRegistry.AddAdapter(&AccessApprovalAdapter{}) +} + +type AccessApprovalAdapter struct { +} + +func (*AccessApprovalAdapter) Weight() int { + return 1 +} + +func (p *AccessApprovalAdapter) PluginType() voluspa.PluginType { + return voluspa.GeneralAdapter +} + +func (p *AccessApprovalAdapter) Type() voluspa.AdapterType { + return voluspa.AdapterType("access_approval") +} + +func (p *AccessApprovalAdapter) Name() string { + return "access_approval" +} + +func (p *AccessApprovalAdapter) ConfigParameters() []string { + return []string{AccessApprovalAdapterParameter} +} + +func (p *AccessApprovalAdapter) SharedLibraryName() string { + return "access_control.so" +} + +func (p *AccessApprovalAdapter) PParams(env *voluspa.ConfigEnvironment) ([]string, error) { + alt, err := env.RemapOptions.ValueByNameAsStringMapInterface(AccessApprovalAdapterParameter) + if err != nil { + return nil, util.FormatError(AccessApprovalAdapterParameter, err) + } + + val, ok := alt["raw"] + if !ok { + return nil, nil + } + + var pparams []string + pparams = append(pparams, strings.Split(val.(string), "\n")...) + + return pparams, nil +} + +func (p *AccessApprovalAdapter) Raw() bool { + return true +} diff --git a/tools/voluspa/adapters/allow_ip.go b/tools/voluspa/adapters/allow_ip.go new file mode 100644 index 00000000000..f7dbf5abdf3 --- /dev/null +++ b/tools/voluspa/adapters/allow_ip.go @@ -0,0 +1,86 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package adapters + +import ( + "bytes" + "fmt" + + "github.com/apache/trafficserver/tools/voluspa" + "github.com/apache/trafficserver/tools/voluspa/adapters/util" +) + +const ( + allowIPAdapterParameter = "allow_ip" + allowIPAdapterParameterParent = "allow_ip_parent" +) + +func init() { + voluspa.AdaptersRegistry.AddAdapter(&AllowIPAdapter{}) +} + +type AllowIPAdapter struct { +} + +func (*AllowIPAdapter) Weight() int { + return 120 +} +func (p *AllowIPAdapter) PluginType() voluspa.PluginType { + return voluspa.ActionAdapter +} + +func (p *AllowIPAdapter) Type() voluspa.AdapterType { + return voluspa.AdapterType("allow_ip") +} + +func (p *AllowIPAdapter) ConfigParameters() []string { + return []string{allowIPAdapterParameter, allowIPAdapterParameterParent} +} + +func (p *AllowIPAdapter) Content(env *voluspa.ConfigEnvironment) (*bytes.Buffer, error) { + switch env.ConfigLocation { + case voluspa.UnknownLocation, voluspa.ChildConfig: + return p.configContent(env, allowIPAdapterParameter) + case voluspa.ParentConfig: + return p.configContent(env, allowIPAdapterParameterParent) + } + return nil, fmt.Errorf("Unhandled configLocation %q", env.ConfigLocation) +} + +func (p *AllowIPAdapter) configContent(env *voluspa.ConfigEnvironment, parameterName string) (*bytes.Buffer, error) { + ips, err := env.RemapOptions.ValueByNameAsSlice(parameterName) + if err != nil { + return nil, nil + } + + if len(ips) == 0 { + return nil, util.FormatError(parameterName, fmt.Errorf("empty IP range speclist")) + } + + // TODO other validation on actual content + + content := &bytes.Buffer{} + for _, ip := range ips { + fmt.Fprintf(content, "@src_ip=%s ", ip) + } + content.WriteString("@action=allow") + + return content, nil +} diff --git a/tools/voluspa/adapters/allow_ip_test.go b/tools/voluspa/adapters/allow_ip_test.go new file mode 100644 index 00000000000..736d81b8938 --- /dev/null +++ b/tools/voluspa/adapters/allow_ip_test.go @@ -0,0 +1,85 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package adapters + +import ( + "testing" + + "github.com/apache/trafficserver/tools/voluspa" +) + +var ( + childIPRange = []string{"17.0.0.0-17.255.255.255", "10.0.0.0-10.255.255.255"} + parentIPRange = []string{"17.253.0.0-17.255.255.255", "10.253.0.0-10.255.255.255"} +) + +func TestIPAllowAdapter(t *testing.T) { + var plugin AllowIPAdapter + ro := &voluspa.RemapOptions{} + env := &voluspa.ConfigEnvironment{RemapOptions: *ro} + + (*ro)[allowIPAdapterParameter] = "" + out, err := plugin.Content(env) + if err == nil && out != nil { + t.Fatalf("Expected error for invalid configuration.") + } + + (*ro)[allowIPAdapterParameter] = []string{} + _, err = plugin.Content(env) + if err == nil { + t.Fatalf("Expected error for invalid configuration.") + } + + (*ro)[allowIPAdapterParameter] = childIPRange + out, err = plugin.Content(env) + if err != nil { + t.Fatalf("Unexpected error. err=%s", err) + } + + expected := "@src_ip=17.0.0.0-17.255.255.255 @src_ip=10.0.0.0-10.255.255.255 @action=allow" + if out.String() != expected { + t.Fatalf("Error: got '%s'; expected '%s'", out, expected) + } + + // "child" config. should still return child version + (*ro)[allowIPAdapterParameterParent] = parentIPRange + out, err = plugin.Content(env) + if err != nil { + t.Fatalf("Unexpected error. err=%s", err) + } + + expected = "@src_ip=17.0.0.0-17.255.255.255 @src_ip=10.0.0.0-10.255.255.255 @action=allow" + if out.String() != expected { + t.Fatalf("Error: got '%s'; expected '%s'", out, expected) + } + + ro = &voluspa.RemapOptions{} + env = &voluspa.ConfigEnvironment{RemapOptions: *ro, ConfigLocation: voluspa.ParentConfig} + (*ro)[allowIPAdapterParameterParent] = []string{"17.0.0.0-17.255.255.255", "10.0.0.0-10.255.255.255"} + out, err = plugin.Content(env) + if err != nil { + t.Fatalf("Unexpected error. err=%s", err) + } + + expected = "@src_ip=17.0.0.0-17.255.255.255 @src_ip=10.0.0.0-10.255.255.255 @action=allow" + if out.String() != expected { + t.Fatalf("Error: got '%s'; expected '%s'", out, expected) + } +} diff --git a/tools/voluspa/adapters/allow_methods.go b/tools/voluspa/adapters/allow_methods.go new file mode 100644 index 00000000000..02ff33a92f6 --- /dev/null +++ b/tools/voluspa/adapters/allow_methods.go @@ -0,0 +1,65 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package adapters + +import ( + "bytes" + "fmt" + + "github.com/apache/trafficserver/tools/voluspa" +) + +func init() { + voluspa.AdaptersRegistry.AddAdapter(&AllowMethodsAdapter{}) +} + +type AllowMethodsAdapter struct { +} + +func (p *AllowMethodsAdapter) Weight() int { + return 140 +} + +func (p *AllowMethodsAdapter) PluginType() voluspa.PluginType { + return voluspa.ActionAdapter +} + +func (p *AllowMethodsAdapter) Type() voluspa.AdapterType { + return voluspa.AdapterType("allow_methods") +} + +func (p *AllowMethodsAdapter) ConfigParameters() []string { + return []string{"allow_methods"} +} + +func (p *AllowMethodsAdapter) Content(env *voluspa.ConfigEnvironment) (*bytes.Buffer, error) { + methods, err := env.RemapOptions.ValueByNameAsSlice("allow_methods") + if err != nil { + return nil, err + } + + content := &bytes.Buffer{} + content.WriteString("@action=allow") + for _, method := range methods { + fmt.Fprintf(content, " @method=%s", method) + } + + return content, nil +} diff --git a/tools/voluspa/adapters/auth_proxy.go b/tools/voluspa/adapters/auth_proxy.go new file mode 100644 index 00000000000..3d9d4be4736 --- /dev/null +++ b/tools/voluspa/adapters/auth_proxy.go @@ -0,0 +1,67 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package adapters + +import ( + "errors" + "fmt" + + "github.com/apache/trafficserver/tools/voluspa" +) + +func init() { + voluspa.AdaptersRegistry.AddAdapter(&AuthProxyAdapter{}) +} + +type AuthProxyAdapter struct { +} + +func (p *AuthProxyAdapter) PluginType() voluspa.PluginType { + return voluspa.GeneralAdapter +} + +func (p *AuthProxyAdapter) Type() voluspa.AdapterType { + return voluspa.AdapterType("auth_proxy") +} + +func (p *AuthProxyAdapter) Name() string { + return "authproxy" +} + +func (p *AuthProxyAdapter) ConfigParameters() []string { + return []string{"authproxy"} +} + +func (p *AuthProxyAdapter) SharedLibraryName() string { + return "authproxy.so" +} + +func (p *AuthProxyAdapter) ConfigLocations() []voluspa.ConfigLocation { + return []voluspa.ConfigLocation{voluspa.ChildConfig} +} + +func (p *AuthProxyAdapter) PParams(env *voluspa.ConfigEnvironment) ([]string, error) { + value, err := env.RemapOptions.ValueByNameAsString(p.Name()) + if err != nil && len(value) == 0 { + return nil, errors.New("Need transform parameter") + } + + return []string{fmt.Sprintf("--auth-transform=%s", value)}, nil +} diff --git a/tools/voluspa/adapters/cache_promote.go b/tools/voluspa/adapters/cache_promote.go new file mode 100644 index 00000000000..16b38f05690 --- /dev/null +++ b/tools/voluspa/adapters/cache_promote.go @@ -0,0 +1,134 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package adapters + +import ( + "fmt" + + "github.com/apache/trafficserver/tools/voluspa" + "github.com/apache/trafficserver/tools/voluspa/adapters/util" +) + +const ( + CachePromoteAdapterParameter = "cache_promote" + CachePromoteAdapterPolicyParameter = "policy" +) + +func init() { + voluspa.AdaptersRegistry.AddAdapter(&CachePromoteAdapter{}) +} + +type CachePromoteAdapter struct { +} + +func (p *CachePromoteAdapter) PluginType() voluspa.PluginType { + return voluspa.GeneralAdapter +} + +func (p *CachePromoteAdapter) Type() voluspa.AdapterType { + return voluspa.AdapterType("cache_promote") +} + +func (p *CachePromoteAdapter) Name() string { + return "cache_promote" +} + +func (p *CachePromoteAdapter) SharedLibraryName() string { + return "cache_promote.so" +} + +func (p *CachePromoteAdapter) Role() string { + return "roles_trafficserver_cache_promote" +} + +func (p *CachePromoteAdapter) ConfigLocations() []voluspa.ConfigLocation { + return []voluspa.ConfigLocation{voluspa.ChildConfig} +} + +func (p *CachePromoteAdapter) ConfigParameters() []string { + return []string{CachePromoteAdapterParameter} +} + +func (p *CachePromoteAdapter) isValidPolicy(policy string) bool { + return policy == "lru" || policy == "chance" +} + +func (p *CachePromoteAdapter) PParams(env *voluspa.ConfigEnvironment) ([]string, error) { + + var alt map[string]interface{} + alt, err := env.RemapOptions.ValueByNameAsStringMapInterface(CachePromoteAdapterParameter) + if err != nil { + return nil, err + } + + if err = p.validateArgs(alt); err != nil { + return nil, err + } + + args, err := p.processArgs(alt) + if err != nil { + return nil, err + } + + return args, nil +} + +func (p *CachePromoteAdapter) validateArgs(alt map[string]interface{}) error { + val := util.ParamToValue(alt["policy"]) + if len(val) == 0 || !p.isValidPolicy(val) { + return util.FormatError(CachePromoteAdapterPolicyParameter, fmt.Errorf("invalid policy specified")) + } + + return nil +} + +func (p *CachePromoteAdapter) processArgs(alt map[string]interface{}) ([]string, error) { + var args []string + args = p.maybeAddArgs(alt["policy"], "policy", args) + args = p.maybeAddArgs(alt["sample"], "sample", args) + + policy := util.ParamToValue(alt["policy"]) + if policy != "lru" { + return args, nil + } + args = p.maybeAddArgs(alt["lru_hits"], "hits", args) + args = p.maybeAddArgs(alt["lru_buckets"], "buckets", args) + + return args, nil +} + +func (p *CachePromoteAdapter) maybeAddArgs(option interface{}, param string, args []string) []string { + value := util.ParamToValue(option) + switch param { + case "policy": + if p.isValidPolicy(value) { + return append(args, fmt.Sprintf("--%s=%s", param, value)) + } + case "sample": + if len(value) > 0 { + return append(args, fmt.Sprintf("--%s=%s%%", param, value)) + } + default: + if len(value) > 0 { + return append(args, fmt.Sprintf("--%s=%s", param, value)) + } + } + return args +} diff --git a/tools/voluspa/adapters/cache_promote_test.go b/tools/voluspa/adapters/cache_promote_test.go new file mode 100644 index 00000000000..99d3e322596 --- /dev/null +++ b/tools/voluspa/adapters/cache_promote_test.go @@ -0,0 +1,107 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package adapters + +import ( + "strings" + "testing" + + "github.com/apache/trafficserver/tools/voluspa" +) + +func TestCachePromoteAdapter(t *testing.T) { + var plugin CachePromoteAdapter + ro := &voluspa.RemapOptions{} + subOptions := make(map[interface{}]interface{}) + (*ro)["cache_promote"] = subOptions + + env := &voluspa.ConfigEnvironment{RemapOptions: *ro} + + _, err := plugin.PParams(env) + if err == nil { + t.Fatalf("Expected error for invalid configuration") + } + + subOptions["policy"] = "notapolicy" + _, err = plugin.PParams(env) + if err == nil { + t.Fatalf("Expected error for invalid configuration") + } + + subOptions["policy"] = "lru" + out, err := plugin.PParams(env) + if err != nil { + t.Fatalf("Unexpected error: %s", err) + } + + expected := "--policy=lru" + if out[0] != expected { + t.Fatalf("Error: got '%s'; expected '%s'", out[0], expected) + } + + subOptions["sample"] = 66 + out, err = plugin.PParams(env) + if err != nil { + t.Fatalf("Unexpected error: %s", err) + } + + joined := strings.Join(out, " ") + expected = "--policy=lru --sample=66%" + if joined != expected { + t.Fatalf("Error: got %q; expected %q", joined, expected) + } + + subOptions["lru_hits"] = 1000 + out, err = plugin.PParams(env) + if err != nil { + t.Fatalf("Unexpected error: %s", err) + } + + joined = strings.Join(out, " ") + expected = "--policy=lru --sample=66% --hits=1000" + if joined != expected { + t.Fatalf("Error: got '%s'; expected '%s'", joined, expected) + } + + subOptions["lru_buckets"] = 10000 + out, err = plugin.PParams(env) + if err != nil { + t.Fatalf("Unexpected error: %s", err) + } + + joined = strings.Join(out, " ") + expected = "--policy=lru --sample=66% --hits=1000 --buckets=10000" + if joined != expected { + t.Fatalf("Error: got %q; expected %q", joined, expected) + } + + // Non-LRU policy's should drop hits and buckets from command-line + subOptions["policy"] = "chance" + out, err = plugin.PParams(env) + if err != nil { + t.Fatalf("Unexpected error: %s", err) + } + + joined = strings.Join(out, " ") + expected = "--policy=chance --sample=66%" + if joined != expected { + t.Fatalf("Error: got %q; expected %q", joined, expected) + } +} diff --git a/tools/voluspa/adapters/cachekey.go b/tools/voluspa/adapters/cachekey.go new file mode 100644 index 00000000000..a349dcd5e19 --- /dev/null +++ b/tools/voluspa/adapters/cachekey.go @@ -0,0 +1,125 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package adapters + +import ( + "fmt" + "strconv" + "strings" + + "github.com/apache/trafficserver/tools/voluspa" +) + +const ( + CacheKeyAdapterParameter = "cachekey" +) + +func init() { + voluspa.AdaptersRegistry.AddAdapter(&CacheKeyAdapter{}) +} + +type CacheKeyAdapter struct { +} + +func (p *CacheKeyAdapter) PluginType() voluspa.PluginType { + return voluspa.GeneralAdapter +} + +func (p *CacheKeyAdapter) Type() voluspa.AdapterType { + return voluspa.AdapterType("cache_key") +} + +func (p *CacheKeyAdapter) SharedLibraryName() string { + return "cachekey.so" +} + +func (p *CacheKeyAdapter) ConfigParameters() []string { + return []string{CacheKeyAdapterParameter} +} + +func (p *CacheKeyAdapter) PParams(env *voluspa.ConfigEnvironment) ([]string, error) { + alt, err := env.RemapOptions.ValueByNameAsStringMapInterface(CacheKeyAdapterParameter) + if err != nil { + return nil, err + } + + args, err := p.processArgs(alt) + if err != nil { + return nil, err + } + + return args, nil +} + +func (p *CacheKeyAdapter) processArgs(alt map[string]interface{}) ([]string, error) { + var args []string + args = p.maybeAddArgs(alt["include_params"], "include-params", args) + args = p.maybeAddArgs(alt["include_cookies"], "include-cookies", args) + args = p.maybeAddArgs(alt["include_headers"], "include-headers", args) + args = p.maybeAddArgs(alt["exclude_params"], "exclude-params", args) + args = p.maybeAddArgs(alt["static_prefix"], "static-prefix", args) + args = p.maybeAddArgs(alt["sort_query"], "sort-params", args) + args = p.maybeAddArgs(alt["remove_all_params"], "remove-all-params", args) + args = p.maybeAddArgs(alt["regex_replace_path"], "capture-path", args) + args = p.maybeAddArgs(alt["regex_replace_path_uri"], "capture-path-uri", args) + args = p.maybeAddArgs(alt["capture_header"], "capture-header", args) + + return args, nil +} + +func (p *CacheKeyAdapter) paramToValue(i interface{}) string { + switch val := i.(type) { + case string: + return val + case int: + return strconv.Itoa(val) + case bool: + return strconv.FormatBool(val) + case []string: + return strings.Join(val, ",") + case []interface{}: + var vals []string + for _, v1 := range i.([]interface{}) { + vS := v1.(string) + vals = append(vals, vS) + } + return strings.Join(vals, ",") + default: + return "" + } +} + +func (p *CacheKeyAdapter) maybeAddArgs(option interface{}, param string, args []string) []string { + + // capture-header needs a separate "--capture-header=" flag for each one when there are multiple + if param == "capture-header" && option != nil { + for _, o := range option.([]interface{}) { + args = append(args, "--capture-header="+o.(string)) + } + return args + } + + value := p.paramToValue(option) + if len(value) > 0 { + return append(args, fmt.Sprintf("--%s=%s", param, value)) + } + + return args +} diff --git a/tools/voluspa/adapters/confremap/base_conf_remap.go b/tools/voluspa/adapters/confremap/base_conf_remap.go new file mode 100644 index 00000000000..4f5a61cac55 --- /dev/null +++ b/tools/voluspa/adapters/confremap/base_conf_remap.go @@ -0,0 +1,63 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package confremap + +import ( + "github.com/apache/trafficserver/tools/voluspa" +) + +func init() { + voluspa.AdaptersRegistry.AddAdapter(&BaseConfRemapAdapter{}) +} + +type BaseConfRemapAdapter struct { +} + +func (*BaseConfRemapAdapter) Weight() int { + return 17 +} + +func (p *BaseConfRemapAdapter) PluginType() voluspa.PluginType { + return voluspa.CompoundAdapter +} + +func (p *BaseConfRemapAdapter) Type() voluspa.AdapterType { + return adapterType +} + +func (p *BaseConfRemapAdapter) CompoundType() voluspa.AdapterType { + return adapterType +} + +func (p *BaseConfRemapAdapter) Name() string { + return "base_conf_remap" +} + +func (p *BaseConfRemapAdapter) SharedLibraryName() string { + return "conf_remap.so" +} + +func (p *BaseConfRemapAdapter) ConfigParameters() []string { + return []string{""} +} + +func (p *BaseConfRemapAdapter) PParams(env *voluspa.ConfigEnvironment) ([]string, error) { + return nil, nil +} diff --git a/tools/voluspa/adapters/confremap/cache_version.go b/tools/voluspa/adapters/confremap/cache_version.go new file mode 100644 index 00000000000..4f0802490ca --- /dev/null +++ b/tools/voluspa/adapters/confremap/cache_version.go @@ -0,0 +1,60 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package confremap + +import ( + "fmt" + + "github.com/apache/trafficserver/tools/voluspa" +) + +func init() { + voluspa.AdaptersRegistry.AddAdapter(&CacheVersionAdapter{}) +} + +type CacheVersionAdapter struct { +} + +func (s *CacheVersionAdapter) PluginType() voluspa.PluginType { + return voluspa.CompoundAdapter +} + +func (s *CacheVersionAdapter) Type() voluspa.AdapterType { + return voluspa.AdapterType("cache_version") +} + +func (s *CacheVersionAdapter) CompoundType() voluspa.AdapterType { + return adapterType +} + +func (s *CacheVersionAdapter) ConfigParameters() []string { + return []string{"cache_version"} +} + +func (s *CacheVersionAdapter) PParams(env *voluspa.ConfigEnvironment) ([]string, error) { + cacheVersion, err := env.RemapOptions.ValueByNameAsInt("cache_version") + if err != nil { + return nil, err + } + + return []string{ + fmt.Sprintf("proxy.config.http.cache.generation=%d", cacheVersion), + }, nil +} diff --git a/tools/voluspa/adapters/confremap/common.go b/tools/voluspa/adapters/confremap/common.go new file mode 100644 index 00000000000..6f64559ac12 --- /dev/null +++ b/tools/voluspa/adapters/confremap/common.go @@ -0,0 +1,24 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package confremap + +import "github.com/apache/trafficserver/tools/voluspa" + +var adapterType = voluspa.AdapterType("conf_remap") diff --git a/tools/voluspa/adapters/confremap/conf_remap.go b/tools/voluspa/adapters/confremap/conf_remap.go new file mode 100644 index 00000000000..a687141e5c2 --- /dev/null +++ b/tools/voluspa/adapters/confremap/conf_remap.go @@ -0,0 +1,107 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package confremap + +import ( + "fmt" + "regexp" + "strconv" + "strings" + + "github.com/apache/trafficserver/tools/voluspa" +) + +func init() { + voluspa.AdaptersRegistry.AddAdapter(&FreeformConfRemapAdapter{}) +} + +type FreeformConfRemapAdapter struct { +} + +func (*FreeformConfRemapAdapter) Weight() int { + return 17 +} + +func (p *FreeformConfRemapAdapter) PluginType() voluspa.PluginType { + return voluspa.CompoundAdapter +} + +func (p *FreeformConfRemapAdapter) Type() voluspa.AdapterType { + return voluspa.AdapterType("raw_conf_remap") +} + +func (p *FreeformConfRemapAdapter) CompoundType() voluspa.AdapterType { + return adapterType +} + +func (p *FreeformConfRemapAdapter) ConfigParameters() []string { + return []string{"conf_remap"} +} + +// CONFIG proxy.config.http.connect_attempts_timeout INT 600 +var confRegexp = regexp.MustCompile(`CONFIG (.*) ([A-Z]+) (.*)`) + +func (p *FreeformConfRemapAdapter) PParams(env *voluspa.ConfigEnvironment) ([]string, error) { + val, err := env.RemapOptions.ValueByNameAsString("conf_remap") + if err != nil { + return nil, err + } + + seen := make(map[string]bool) + + var params []string + + lines := strings.Split(val, "\n") + for _, line := range lines { + line = strings.TrimSpace(line) + if line == "" { + continue + } + + // eg CONFIG proxy.config.http.connect_attempts_timeout INT 600 + vals := confRegexp.FindStringSubmatch(line) + if vals == nil { + return nil, fmt.Errorf("%s is not properly formed", line) + } + + if _, ok := seen[line]; ok { + return nil, fmt.Errorf("Already seen key '%s'", vals[1]) + } + seen[line] = true + + switch vals[2] { + case "INT": + _, err := strconv.Atoi(vals[3]) + if err != nil { + return nil, fmt.Errorf("Value is not of appropriate type; in=%s type=%s line='%s'", vals[3], vals[2], line) + } + case "FLOAT": + _, err := strconv.ParseFloat(vals[3], 64) + if err != nil { + return nil, fmt.Errorf("Value is not of appropriate type; in=%s type=%s line='%s'", vals[3], vals[2], line) + } + case "STRING": + default: + return nil, fmt.Errorf("Type is unknown/unhandled; in=%s type=%s line='%s'", vals[3], vals[2], line) + } + params = append(params, fmt.Sprintf("%s=%s", vals[1], vals[3])) + } + return params, nil +} diff --git a/tools/voluspa/adapters/confremap/disable_cache.go b/tools/voluspa/adapters/confremap/disable_cache.go new file mode 100644 index 00000000000..06398e78476 --- /dev/null +++ b/tools/voluspa/adapters/confremap/disable_cache.go @@ -0,0 +1,51 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package confremap + +import ( + "github.com/apache/trafficserver/tools/voluspa" +) + +func init() { + voluspa.AdaptersRegistry.AddAdapter(&DisableCacheAdapter{}) +} + +type DisableCacheAdapter struct { +} + +func (p *DisableCacheAdapter) PluginType() voluspa.PluginType { + return voluspa.CompoundAdapter +} + +func (p *DisableCacheAdapter) Type() voluspa.AdapterType { + return voluspa.AdapterType("disable_cache") +} + +func (p *DisableCacheAdapter) CompoundType() voluspa.AdapterType { + return adapterType +} + +func (p *DisableCacheAdapter) ConfigParameters() []string { + return []string{"disable_cache"} +} + +func (p *DisableCacheAdapter) PParams(env *voluspa.ConfigEnvironment) ([]string, error) { + return []string{"proxy.config.http.cache.http=0"}, nil +} diff --git a/tools/voluspa/adapters/confremap/negative_caching.go b/tools/voluspa/adapters/confremap/negative_caching.go new file mode 100644 index 00000000000..9c9af2a1c49 --- /dev/null +++ b/tools/voluspa/adapters/confremap/negative_caching.go @@ -0,0 +1,68 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package confremap + +import ( + "fmt" + + "github.com/apache/trafficserver/tools/voluspa" + "github.com/apache/trafficserver/tools/voluspa/adapters/util" +) + +const ( + NegativeCachingAdapterParameterTTL = "negative_ttl" +) + +func init() { + voluspa.AdaptersRegistry.AddAdapter(&NegativeCachingAdapter{}) +} + +type NegativeCachingAdapter struct { +} + +func (p *NegativeCachingAdapter) PluginType() voluspa.PluginType { + return voluspa.CompoundAdapter +} + +func (p *NegativeCachingAdapter) Type() voluspa.AdapterType { + return voluspa.AdapterType("negative_caching") +} + +func (p *NegativeCachingAdapter) CompoundType() voluspa.AdapterType { + return adapterType +} + +func (p *NegativeCachingAdapter) ConfigParameters() []string { + return []string{NegativeCachingAdapterParameterTTL} +} + +func (p *NegativeCachingAdapter) PParams(env *voluspa.ConfigEnvironment) ([]string, error) { + duration, err := env.RemapOptions.ValueByNameAsString(NegativeCachingAdapterParameterTTL) + if err != nil { + return nil, err + } + + seconds, err := util.DurationToSeconds(duration) + if err != nil { + return nil, err + } + + return []string{fmt.Sprintf("proxy.config.http.negative_caching_lifetime=%s", seconds)}, nil +} diff --git a/tools/voluspa/adapters/confremap/preserve_host_header.go b/tools/voluspa/adapters/confremap/preserve_host_header.go new file mode 100644 index 00000000000..a2d35f0d7c3 --- /dev/null +++ b/tools/voluspa/adapters/confremap/preserve_host_header.go @@ -0,0 +1,82 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package confremap + +import ( + "fmt" + + "github.com/apache/trafficserver/tools/voluspa" + "github.com/apache/trafficserver/tools/voluspa/adapters/util" +) + +const ( + OriginHostHeaderAdapterParameter = "origin_host_header" +) + +func init() { + voluspa.AdaptersRegistry.AddAdapter(&PreserveHostHeaderAdapter{}) +} + +type PreserveHostHeaderAdapter struct { +} + +func (*PreserveHostHeaderAdapter) Weight() int { + return 25 +} + +func (p *PreserveHostHeaderAdapter) PluginType() voluspa.PluginType { + return voluspa.CompoundAdapter +} + +func (p *PreserveHostHeaderAdapter) Type() voluspa.AdapterType { + return voluspa.AdapterType("preserve_host_header") +} + +func (p *PreserveHostHeaderAdapter) CompoundType() voluspa.AdapterType { + return adapterType +} + +func (p *PreserveHostHeaderAdapter) ConfigLocations() []voluspa.ConfigLocation { + return []voluspa.ConfigLocation{voluspa.ParentConfig} +} + +func (p *PreserveHostHeaderAdapter) ConfigParameters() []string { + return []string{OriginHostHeaderAdapterParameter} +} + +func (p *PreserveHostHeaderAdapter) PParams(env *voluspa.ConfigEnvironment) ([]string, error) { + intval := 0 + if env.RemapOptions.HasOption(OriginHostHeaderAdapterParameter) { + ohhValue, err := env.RemapOptions.ValueByNameAsString(OriginHostHeaderAdapterParameter) + if err != nil { + return nil, util.FormatError(OriginHostHeaderAdapterParameter, err) + } + switch ohhValue { + case "alias": + intval = 1 + case "origin": + intval = 0 + default: + return nil, util.FormatError(OriginHostHeaderAdapterParameter, fmt.Errorf("unknown value %q", ohhValue)) + } + } + + return []string{fmt.Sprintf("proxy.config.url_remap.pristine_host_hdr=%d", intval)}, nil +} diff --git a/tools/voluspa/adapters/confremap/transaction_timeout.go b/tools/voluspa/adapters/confremap/transaction_timeout.go new file mode 100644 index 00000000000..db462cf17ae --- /dev/null +++ b/tools/voluspa/adapters/confremap/transaction_timeout.go @@ -0,0 +1,68 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package confremap + +import ( + "fmt" + + "github.com/apache/trafficserver/tools/voluspa" + "github.com/apache/trafficserver/tools/voluspa/adapters/util" +) + +func init() { + voluspa.AdaptersRegistry.AddAdapter(&TransactionTimeoutAdapter{}) +} + +type TransactionTimeoutAdapter struct { +} + +func (s *TransactionTimeoutAdapter) PluginType() voluspa.PluginType { + return voluspa.CompoundAdapter +} + +func (s *TransactionTimeoutAdapter) Type() voluspa.AdapterType { + return voluspa.AdapterType("transaction_timeout") +} + +func (s *TransactionTimeoutAdapter) CompoundType() voluspa.AdapterType { + return adapterType +} + +func (s *TransactionTimeoutAdapter) ConfigParameters() []string { + return []string{"transaction_timeout"} +} + +func (s *TransactionTimeoutAdapter) PParams(env *voluspa.ConfigEnvironment) ([]string, error) { + duration, err := env.RemapOptions.ValueByNameAsString("transaction_timeout") + if err != nil { + return nil, err + } + + seconds, err := util.DurationToSeconds(duration) + if err != nil { + return nil, err + } + + return []string{ + fmt.Sprintf("proxy.config.http.connect_attempts_timeout=%s", seconds), + fmt.Sprintf("proxy.config.http.transaction_no_activity_timeout_in=%s", seconds), + fmt.Sprintf("proxy.config.http.transaction_no_activity_timeout_out=%s", seconds), + }, nil +} diff --git a/tools/voluspa/adapters/deny_methods.go b/tools/voluspa/adapters/deny_methods.go new file mode 100644 index 00000000000..580447dd67f --- /dev/null +++ b/tools/voluspa/adapters/deny_methods.go @@ -0,0 +1,69 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package adapters + +import ( + "bytes" + "fmt" + + "github.com/apache/trafficserver/tools/voluspa" +) + +func init() { + voluspa.AdaptersRegistry.AddAdapter(&DenyMethodsAdapter{}) +} + +type DenyMethodsAdapter struct { +} + +func (*DenyMethodsAdapter) Weight() int { + return 150 +} + +func (p *DenyMethodsAdapter) PluginType() voluspa.PluginType { + return voluspa.ActionAdapter +} + +func (p *DenyMethodsAdapter) Type() voluspa.AdapterType { + return voluspa.AdapterType("deny_methods") +} + +func (p *DenyMethodsAdapter) ConfigLocations() []voluspa.ConfigLocation { + return []voluspa.ConfigLocation{voluspa.ChildConfig} +} + +func (p *DenyMethodsAdapter) ConfigParameters() []string { + return []string{"deny_methods"} +} + +func (p *DenyMethodsAdapter) Content(env *voluspa.ConfigEnvironment) (*bytes.Buffer, error) { + methods, err := env.RemapOptions.ValueByNameAsSlice("deny_methods") + if err != nil { + return nil, err + } + + content := &bytes.Buffer{} + content.WriteString("@action=deny") + for _, method := range methods { + fmt.Fprintf(content, " @method=%s", method) + } + + return content, nil +} diff --git a/tools/voluspa/adapters/escalate.go b/tools/voluspa/adapters/escalate.go new file mode 100644 index 00000000000..d4e0997d07a --- /dev/null +++ b/tools/voluspa/adapters/escalate.go @@ -0,0 +1,150 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package adapters + +import ( + "fmt" + "strconv" + "strings" + + "github.com/apache/trafficserver/tools/voluspa" +) + +const ( + EscalateAdapterFailoverParameter = "failover" +) + +func init() { + voluspa.AdaptersRegistry.AddAdapter(&EscalateAdapter{}) +} + +type EscalateAdapter struct { +} + +func (p *EscalateAdapter) PluginType() voluspa.PluginType { + return voluspa.GeneralAdapter +} + +func (p *EscalateAdapter) Type() voluspa.AdapterType { + return voluspa.AdapterType("escalate") +} + +func (p *EscalateAdapter) Name() string { + return "escalate" +} + +func (p *EscalateAdapter) SharedLibraryName() string { + return "escalate.so" +} + +func (p *EscalateAdapter) ConfigParameters() []string { + return []string{EscalateAdapterFailoverParameter} +} + +func (p *EscalateAdapter) ConfigLocations() []voluspa.ConfigLocation { + return []voluspa.ConfigLocation{voluspa.ParentConfig} +} + +func (p *EscalateAdapter) PParams(env *voluspa.ConfigEnvironment) ([]string, error) { + alt, err := env.RemapOptions.ValueByNameAsStringMapInterface(EscalateAdapterFailoverParameter) + if err != nil { + return nil, err + } + + if err = p.validateArgs(alt); err != nil { + return nil, err + } + + args, err := p.processArgs(alt) + if err != nil { + return nil, err + } + + return args, nil +} + +func (p *EscalateAdapter) validateArgs(alt map[string]interface{}) error { + domain := p.paramToValue(alt["domain"]) + statusCodes := p.paramToValue(alt["status_codes"]) + if len(domain) == 0 || len(statusCodes) == 0 { + return fmt.Errorf("%q and %q are required parameters", "domain", "status_codes") + } + + value := p.paramToValue(alt["host_header"]) + if len(value) > 0 && (value != "alias" && value != "origin") { + return fmt.Errorf("invalid 'host_header' option: %q", value) + } + + return nil +} + +func convertStatusCodes(in interface{}) []string { + switch val := in.(type) { + case []string: + return val + case []interface{}: + var vals []string + for _, v1 := range in.([]interface{}) { + vS := strconv.Itoa(v1.(int)) + vals = append(vals, vS) + } + return vals + } + return nil +} + +func (p *EscalateAdapter) processArgs(alt map[string]interface{}) ([]string, error) { + var args []string + + { + value := p.paramToValue(alt["host_header"]) + if len(value) > 0 && value == "origin" { + args = append(args, "--pristine") + } + } + + statusCodes := convertStatusCodes(alt["status_codes"]) + domain := p.paramToValue(alt["domain"]) + + args = append(args, fmt.Sprintf("%s:%s", strings.Join(statusCodes, ","), domain)) + + return args, nil +} + +func (p *EscalateAdapter) paramToValue(i interface{}) string { + switch val := i.(type) { + case string: + return val + case bool: + return strconv.FormatBool(val) + + case []string: + return strings.Join(val, ",") + case []interface{}: + var vals []string + for _, v1 := range i.([]interface{}) { + vS := strconv.Itoa(v1.(int)) + vals = append(vals, vS) + } + return strings.Join(vals, ",") + default: + return "" + } +} diff --git a/tools/voluspa/adapters/escalate_test.go b/tools/voluspa/adapters/escalate_test.go new file mode 100644 index 00000000000..83de0032c07 --- /dev/null +++ b/tools/voluspa/adapters/escalate_test.go @@ -0,0 +1,86 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package adapters + +import ( + "testing" + + "github.com/apache/trafficserver/tools/voluspa" +) + +func TestEscalateAdapter(t *testing.T) { + var plugin EscalateAdapter + ro := &voluspa.RemapOptions{} + + subOptions := make(map[interface{}]interface{}) + (*ro)[EscalateAdapterFailoverParameter] = subOptions + + env := &voluspa.ConfigEnvironment{RemapOptions: *ro} + + _, err := plugin.PParams(env) + if err == nil { + t.Fatalf("Expected error for invalid configuration.") + } + + subOptions["domain"] = "" + _, err = plugin.PParams(env) + if err == nil { + t.Fatalf("Expected error for invalid configuration.") + } + + subOptions["domain"] = "domain.example.com" + _, err = plugin.PParams(env) + if err == nil { + t.Fatalf("Expected error for invalid configuration.") + } + + subOptions["status_codes"] = []string{} + _, err = plugin.PParams(env) + if err == nil { + t.Fatalf("Expected error for invalid configuration.") + } + + subOptions["status_codes"] = []string{"401", "403"} + out, err := plugin.PParams(env) + if err != nil { + t.Fatalf("Unexpected error: %s", err) + } + + expected := "401,403:domain.example.com" + if out[0] != expected { + t.Fatalf("Error: got %q; expected %q", out[0], expected) + } + + subOptions["host_header"] = "origin" + out, err = plugin.PParams(env) + if err != nil { + t.Fatalf("Unexpected error: %s", err) + } + + expected = "--pristine" + if out[0] != expected { + t.Fatalf("Error: got %q; expected %q", out[0], expected) + } + + expected = "401,403:domain.example.com" + if out[1] != expected { + t.Fatalf("Error: got %q; expected %q", out[1], expected) + } +} diff --git a/tools/voluspa/adapters/gzip.go b/tools/voluspa/adapters/gzip.go new file mode 100644 index 00000000000..ee486c552f0 --- /dev/null +++ b/tools/voluspa/adapters/gzip.go @@ -0,0 +1,198 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package adapters + +import ( + "bytes" + "fmt" + "sort" + "strings" + + "github.com/apache/trafficserver/tools/voluspa" +) + +const ( + GzipAdapterParameterCompressibleContentType = "static_origin_compressible_content_type" + GzipAdapterParameter = "edge_compression" + GzipAdapterParameterEnabled = "enabled" + GzipAdapterParameterAlgorithms = "algorithms" + GzipAdapterParameterRemoveAcceptEncoding = "remove_accept_encoding" + GzipAdapterParameterFlush = "flush" + GzipAdapterParameterMinimumContentLength = "minimum_content_length" +) + +var ( + defaultCompressibleTypes = []string{ + "*font*", + "*javascript", + "*json", + "*ml;*", + "*mpegURL", + "*mpegurl", + "*otf", + "*ttf", + "*type", + "*xml", + "application/eot", + "application/pkix-crl", + "application/x-httpd-cgi", + "application/x-perl", + "image/vnd.microsoft.icon", + "image/x-icon", + "text/*", + } +) + +func init() { + voluspa.AdaptersRegistry.AddAdapter(&GzipAdapter{}) +} + +var ( + GzipDefaultAlgorithms = []string{"gzip"} +) + +type GzipAdapter struct { +} + +func (p *GzipAdapter) PluginType() voluspa.PluginType { + return voluspa.GeneralAdapter +} + +func (p *GzipAdapter) Type() voluspa.AdapterType { + return voluspa.AdapterType("gzip") +} + +func (p *GzipAdapter) Name() string { + return "gzip" +} + +func (p *GzipAdapter) SharedLibraryName() string { + return "gzip.so" +} + +func (p *GzipAdapter) ConfigLocations() []voluspa.ConfigLocation { + return []voluspa.ConfigLocation{voluspa.ParentConfig} +} + +func (p *GzipAdapter) ConfigParameters() []string { + return []string{GzipAdapterParameter} +} + +func (p *GzipAdapter) Content(env *voluspa.ConfigEnvironment) (*bytes.Buffer, error) { + content := &bytes.Buffer{} + + config, err := env.RemapOptions.ValueByNameAsStringMapInterface(GzipAdapterParameter) + if err != nil { + return nil, err + } + + { + value, hasOption := config[GzipAdapterParameterEnabled] + if hasOption { + bvalue, ok := value.(bool) + if !ok { + return nil, fmt.Errorf("invalid 'enabled' value (%s): %t", value, value) + } + + if !bvalue { + return content, nil + } + } + } + + var algorithms []string + avalue, hasOption := config[GzipAdapterParameterAlgorithms] + if hasOption { + sliceValue, ok := avalue.([]interface{}) + if !ok { + return nil, fmt.Errorf("invalid value for %q: (%s) %t", GzipAdapterParameterAlgorithms, avalue, avalue) + } + + if len(sliceValue) > 0 { + for _, a := range sliceValue { + value := a.(string) + switch value { + case "gzip": + fallthrough + case "br": + algorithms = append(algorithms, value) + default: + return nil, fmt.Errorf("invalid value for %q: (%s)", GzipAdapterParameterAlgorithms, value) + } + } + } + } else { + algorithms = GzipDefaultAlgorithms + } + sort.Sort(sort.StringSlice(algorithms)) + + var compressibleTypes []string + params, found := config[GzipAdapterParameterCompressibleContentType] + if found { + typesSlice, ok := params.([]interface{}) + if !ok { + return nil, fmt.Errorf("invalid value for %q: (%s) %t", GzipAdapterParameterCompressibleContentType, params, params) + } + + for _, a := range typesSlice { + compressibleTypes = append(compressibleTypes, a.(string)) + } + } else { + compressibleTypes = defaultCompressibleTypes + } + + sort.Sort(sort.StringSlice(compressibleTypes)) + + content.WriteString(`enabled true +cache false +`) + + value, hasOption := config[GzipAdapterParameterRemoveAcceptEncoding] + if hasOption { + on, ok := value.(bool) + if ok && on { + content.WriteString("remove-accept-encoding true\n") + } + } + + value, hasOption = config[GzipAdapterParameterFlush] + if hasOption { + on, ok := value.(bool) + if ok && on { + content.WriteString("flush true\n") + } + } + + value, hasOption = config[GzipAdapterParameterMinimumContentLength] + if hasOption { + minLen, ok := value.(int) + if ok && minLen > 0 { + content.WriteString(fmt.Sprintf("minimum-content-length %d\n", minLen)) + } + } + + content.WriteString(fmt.Sprintf("supported-algorithms %s\n", strings.Join(algorithms, ","))) + + for _, param := range compressibleTypes { + content.WriteString(fmt.Sprintf("compressible-content-type %s\n", param)) + } + + return content, nil +} diff --git a/tools/voluspa/adapters/headerrewrite/add_header.go b/tools/voluspa/adapters/headerrewrite/add_header.go new file mode 100644 index 00000000000..ec5fcce7f54 --- /dev/null +++ b/tools/voluspa/adapters/headerrewrite/add_header.go @@ -0,0 +1,75 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package headerrewrite + +import ( + "bytes" + "fmt" + + "github.com/apache/trafficserver/tools/voluspa" + "github.com/apache/trafficserver/tools/voluspa/adapters/util" +) + +const ( + AddHeaderAdapterParameter = "add_header" + AddHeaderAdapterOriginParameter = "add_header_origin" +) + +func init() { + voluspa.AdaptersRegistry.AddAdapter(&AddHeaderAdapter{}) +} + +type AddHeaderAdapter struct { +} + +func (p *AddHeaderAdapter) PluginType() voluspa.PluginType { + return voluspa.CompoundAdapter +} + +func (p *AddHeaderAdapter) Type() voluspa.AdapterType { + return voluspa.AdapterType("add_header") +} + +func (p *AddHeaderAdapter) CompoundType() voluspa.AdapterType { + return adapterType +} + +func (p *AddHeaderAdapter) ConfigParameters() []string { + return []string{AddHeaderAdapterParameter, AddHeaderAdapterOriginParameter} +} + +func (p *AddHeaderAdapter) Content(env *voluspa.ConfigEnvironment) (*bytes.Buffer, error) { + header, _ := env.RemapOptions.ValueByNameAsString(AddHeaderAdapterParameter) + + var buf bytes.Buffer + if len(header) > 0 { + buf.WriteString(util.FormatSimpleHeaderRewrite("READ_RESPONSE_HDR_HOOK", fmt.Sprintf("add-header %s", header))) + } + + header, _ = env.RemapOptions.ValueByNameAsString(AddHeaderAdapterOriginParameter) + if len(header) > 0 { + if buf.Len() > 0 { + buf.WriteByte('\n') + } + buf.WriteString(util.FormatSimpleHeaderRewrite("SEND_REQUEST_HDR_HOOK", fmt.Sprintf("add-header %s", header))) + } + + return &buf, nil +} diff --git a/tools/voluspa/adapters/headerrewrite/base_header_rewrite.go b/tools/voluspa/adapters/headerrewrite/base_header_rewrite.go new file mode 100644 index 00000000000..9c2daed2288 --- /dev/null +++ b/tools/voluspa/adapters/headerrewrite/base_header_rewrite.go @@ -0,0 +1,65 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package headerrewrite + +import ( + "bytes" + + "github.com/apache/trafficserver/tools/voluspa" +) + +func init() { + voluspa.AdaptersRegistry.AddAdapter(&BaseHeaderRewriteAdapter{}) +} + +type BaseHeaderRewriteAdapter struct { +} + +func (*BaseHeaderRewriteAdapter) Weight() int { + return 25 +} + +func (p *BaseHeaderRewriteAdapter) PluginType() voluspa.PluginType { + return voluspa.CompoundAdapter +} + +func (p *BaseHeaderRewriteAdapter) Type() voluspa.AdapterType { + return adapterType +} + +func (p *BaseHeaderRewriteAdapter) CompoundType() voluspa.AdapterType { + return adapterType +} + +func (p *BaseHeaderRewriteAdapter) Name() string { + return "hdrs" +} + +func (p *BaseHeaderRewriteAdapter) SharedLibraryName() string { + return "header_rewrite.so" +} + +func (p *BaseHeaderRewriteAdapter) ConfigParameters() []string { + return []string{""} +} + +func (p *BaseHeaderRewriteAdapter) Content(env *voluspa.ConfigEnvironment) (*bytes.Buffer, error) { + return &bytes.Buffer{}, nil +} diff --git a/tools/voluspa/adapters/headerrewrite/common.go b/tools/voluspa/adapters/headerrewrite/common.go new file mode 100644 index 00000000000..dd14765f974 --- /dev/null +++ b/tools/voluspa/adapters/headerrewrite/common.go @@ -0,0 +1,24 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package headerrewrite + +import "github.com/apache/trafficserver/tools/voluspa" + +var adapterType = voluspa.AdapterType("header_rewrite") diff --git a/tools/voluspa/adapters/headerrewrite/content_type_forge.go b/tools/voluspa/adapters/headerrewrite/content_type_forge.go new file mode 100644 index 00000000000..b92b1a94f00 --- /dev/null +++ b/tools/voluspa/adapters/headerrewrite/content_type_forge.go @@ -0,0 +1,88 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package headerrewrite + +import ( + "bytes" + "fmt" + "sort" + + "github.com/apache/trafficserver/tools/voluspa" + "github.com/apache/trafficserver/tools/voluspa/adapters/util" +) + +const ( + ContentTypeForgeAdapterParameter = "content_type_forge" +) + +func init() { + voluspa.AdaptersRegistry.AddAdapter(&ContentTypeForgeAdapter{}) +} + +type ContentTypeForgeAdapter struct { +} + +func (p *ContentTypeForgeAdapter) PluginType() voluspa.PluginType { + return voluspa.CompoundAdapter +} + +func (p *ContentTypeForgeAdapter) Type() voluspa.AdapterType { + return voluspa.AdapterType("content_type_forge") +} + +func (p *ContentTypeForgeAdapter) CompoundType() voluspa.AdapterType { + return adapterType +} + +func (p *ContentTypeForgeAdapter) Name() string { + return "" +} + +func (p *ContentTypeForgeAdapter) ConfigParameters() []string { + return []string{ContentTypeForgeAdapterParameter} +} + +func (p *ContentTypeForgeAdapter) Content(env *voluspa.ConfigEnvironment) (*bytes.Buffer, error) { + content := &bytes.Buffer{} + + contentTypes, err := env.RemapOptions.ValueByNameAsStringMapString(ContentTypeForgeAdapterParameter) + if err != nil { + return nil, util.FormatError(ContentTypeForgeAdapterParameter, err) + } + + var keys []string + for k := range contentTypes { + keys = append(keys, k) + } + sort.Strings(keys) + + for _, regex := range keys { + contentType := contentTypes[regex] + content.WriteString(fmt.Sprintf( + `cond %%{SEND_RESPONSE_HDR_HOOK} [AND] + cond %%{PATH} /%s/ + set-header Content-Type %s`, + regex, contentType, + )) + content.WriteString("\n") + } + + return content, nil +} diff --git a/tools/voluspa/adapters/headerrewrite/defaultttl.go b/tools/voluspa/adapters/headerrewrite/defaultttl.go new file mode 100644 index 00000000000..6cadc29d2cd --- /dev/null +++ b/tools/voluspa/adapters/headerrewrite/defaultttl.go @@ -0,0 +1,83 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package headerrewrite + +import ( + "bytes" + "fmt" + + "github.com/apache/trafficserver/tools/voluspa" + "github.com/apache/trafficserver/tools/voluspa/adapters/util" +) + +const ( + DefaultTTLAdapterParameter = "default_ttl" +) + +func init() { + voluspa.AdaptersRegistry.AddAdapter(&DefaultTTLAdapter{}) +} + +type DefaultTTLAdapter struct { +} + +func (p *DefaultTTLAdapter) PluginType() voluspa.PluginType { + return voluspa.CompoundAdapter +} + +func (p *DefaultTTLAdapter) Type() voluspa.AdapterType { + return voluspa.AdapterType("default_ttl") +} + +func (p *DefaultTTLAdapter) CompoundType() voluspa.AdapterType { + return adapterType +} + +func (p *DefaultTTLAdapter) Name() string { + return "" +} + +func (p *DefaultTTLAdapter) ConfigParameters() []string { + return []string{DefaultTTLAdapterParameter} +} + +func (p *DefaultTTLAdapter) Content(env *voluspa.ConfigEnvironment) (*bytes.Buffer, error) { + content := &bytes.Buffer{} + + cacheControl, err := env.RemapOptions.ValueByNameAsString(DefaultTTLAdapterParameter) + if err != nil { + return nil, util.FormatError(DefaultTTLAdapterParameter, err) + } + + // translate a simple duration to a max-age public directive + if maxAge, err := util.DurationToSeconds(cacheControl); err == nil { + cacheControl = fmt.Sprintf("max-age=%s, public", maxAge) + } + + content.WriteString(fmt.Sprintf( + `cond %%{READ_RESPONSE_HDR_HOOK} [AND] +cond %%{HEADER:Cache-Control} ="" [AND] +%s + set-header Cache-Control %q`, + conditionStatusCodeSuccess, cacheControl, + )) + + return content, nil +} diff --git a/tools/voluspa/adapters/headerrewrite/defaultttl_test.go b/tools/voluspa/adapters/headerrewrite/defaultttl_test.go new file mode 100644 index 00000000000..305f3d6a833 --- /dev/null +++ b/tools/voluspa/adapters/headerrewrite/defaultttl_test.go @@ -0,0 +1,97 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package headerrewrite + +import ( + "errors" + "fmt" + "strings" + "testing" + + "github.com/apache/trafficserver/tools/voluspa" +) + +func TestDefaultTTL(t *testing.T) { + + plugin := &DefaultTTLAdapter{} + + if plugin.Type() != "default_ttl" { + t.Errorf("Type() returned unexpected %q", plugin.Type()) + } + + if fmt.Sprintf("%v", plugin.ConfigParameters()) != "[default_ttl]" { + t.Errorf("ConfigParameters() returned unexpected %v", plugin.ConfigParameters()) + } + + for i, tt := range []struct { + param interface{} + expected string + err error + }{ + { + param: nil, + err: errors.New(`option "default_ttl": value for "default_ttl" is not a string: `), + }, + { + param: true, + err: errors.New(`option "default_ttl": value for "default_ttl" is not a string: true`), + }, + { + param: 42, + err: errors.New(`option "default_ttl": value for "default_ttl" is not a string: 42`), + }, + { + param: "5d", + expected: ` +cond %{READ_RESPONSE_HDR_HOOK} [AND] +cond %{HEADER:Cache-Control} ="" [AND] +cond %{STATUS} >199 [AND] +cond %{STATUS} <400 + set-header Cache-Control "max-age=432000, public"`, + }, + { + param: "no-cache, no-store, must-revalidate, stale-while-revalidate=30", + expected: ` +cond %{READ_RESPONSE_HDR_HOOK} [AND] +cond %{HEADER:Cache-Control} ="" [AND] +cond %{STATUS} >199 [AND] +cond %{STATUS} <400 + set-header Cache-Control "no-cache, no-store, must-revalidate, stale-while-revalidate=30"`, + }, + } { + + ro := &voluspa.RemapOptions{} + env := &voluspa.ConfigEnvironment{RemapOptions: *ro} + + t.Run(fmt.Sprintf("%d-%v", i, tt.param), func(t *testing.T) { + + (*ro)[DefaultTTLAdapterParameter] = tt.param + got, err := plugin.Content(env) + if fmt.Sprintf("%v", err) != fmt.Sprintf("%v", tt.err) { + t.Errorf("Content() error expected `%v` got: %v", tt.err, err) + return + } + expected := strings.TrimPrefix(tt.expected, "\n") + if err == nil && got.String() != expected { + t.Errorf("Content() buffer expected:\n%s\ngot:\n%s", expected, got.String()) + } + }) + } +} diff --git a/tools/voluspa/adapters/headerrewrite/dscp.go b/tools/voluspa/adapters/headerrewrite/dscp.go new file mode 100644 index 00000000000..94618995503 --- /dev/null +++ b/tools/voluspa/adapters/headerrewrite/dscp.go @@ -0,0 +1,92 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package headerrewrite + +import ( + "bytes" + "fmt" + + "github.com/apache/trafficserver/tools/voluspa" + "github.com/apache/trafficserver/tools/voluspa/adapters/util" +) + +const ( + DSCPAdapterParameterPriority = "priority" +) + +func init() { + voluspa.AdaptersRegistry.AddAdapter(&DSCPAdapter{}) +} + +type DSCPAdapter struct { +} + +func (p *DSCPAdapter) PluginType() voluspa.PluginType { + return voluspa.CompoundAdapter +} + +func (p *DSCPAdapter) Type() voluspa.AdapterType { + return voluspa.AdapterType("dscp") +} + +func (p *DSCPAdapter) CompoundType() voluspa.AdapterType { + return adapterType +} + +func (p *DSCPAdapter) ConfigParameters() []string { + return []string{DSCPAdapterParameterPriority} +} + +func (p *DSCPAdapter) ConfigLocations() []voluspa.ConfigLocation { + return []voluspa.ConfigLocation{voluspa.ChildConfig} +} + +func (p *DSCPAdapter) Content(env *voluspa.ConfigEnvironment) (*bytes.Buffer, error) { + content := &bytes.Buffer{} + value, err := env.RemapOptions.ValueByNameAsString(DSCPAdapterParameterPriority) + if err != nil { + return nil, err + } + + // http://www.tucny.com/Home/dscp-tos + var dscpvalue int + switch value { + case "background": + // Queue 1 - Low-Priority Data - CS1 + dscpvalue = 8 + case "foreground": + // Queue 0 - Best-Effort - AF31, AF32, AF33, AF11, AF12, AF13 + // + // AF11 = High-Throughput Data - iTunes Downloads (App Store, Songs, etc.) + dscpvalue = 10 + case "streamingaudio": + // AF31 = Multimedia Streaming - HTTP Live Streaming - Streaming Audio + dscpvalue = 26 + case "streamingvideo": + // AF32 = Multimedia Streaming - HTTP Live Streaming - Streaming Video + dscpvalue = 28 + default: + return nil, util.FormatError(DSCPAdapterParameterPriority, fmt.Errorf("Unhandled priority %q", value)) + } + + content.WriteString(util.FormatSimpleHeaderRewrite("REMAP_PSEUDO_HOOK", fmt.Sprintf("set-conn-dscp %d", dscpvalue))) + + return content, nil +} diff --git a/tools/voluspa/adapters/headerrewrite/forcettl.go b/tools/voluspa/adapters/headerrewrite/forcettl.go new file mode 100644 index 00000000000..c70b91db394 --- /dev/null +++ b/tools/voluspa/adapters/headerrewrite/forcettl.go @@ -0,0 +1,82 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package headerrewrite + +import ( + "bytes" + "fmt" + + "github.com/apache/trafficserver/tools/voluspa" + "github.com/apache/trafficserver/tools/voluspa/adapters/util" +) + +const ( + ForceTTLAdapterParameter = "force_ttl" +) + +func init() { + voluspa.AdaptersRegistry.AddAdapter(&ForceTTLAdapter{}) +} + +type ForceTTLAdapter struct { +} + +func (p *ForceTTLAdapter) PluginType() voluspa.PluginType { + return voluspa.CompoundAdapter +} + +func (p *ForceTTLAdapter) Type() voluspa.AdapterType { + return voluspa.AdapterType("force_ttl") +} + +func (p *ForceTTLAdapter) CompoundType() voluspa.AdapterType { + return adapterType +} + +func (p *ForceTTLAdapter) Name() string { + return "" +} + +func (p *ForceTTLAdapter) ConfigParameters() []string { + return []string{ForceTTLAdapterParameter} +} + +func (p *ForceTTLAdapter) Content(env *voluspa.ConfigEnvironment) (*bytes.Buffer, error) { + content := &bytes.Buffer{} + + cacheControl, err := env.RemapOptions.ValueByNameAsString(ForceTTLAdapterParameter) + if err != nil { + return nil, util.FormatError(ForceTTLAdapterParameter, err) + } + + // translate a simple duration to a max-age public directive + if maxAge, err := util.DurationToSeconds(cacheControl); err == nil { + cacheControl = fmt.Sprintf("max-age=%s, public", maxAge) + } + + content.WriteString(fmt.Sprintf( + `cond %%{READ_RESPONSE_HDR_HOOK} [AND] +%s + set-header Cache-Control %q`, + conditionStatusCodeSuccess, cacheControl, + )) + + return content, nil +} diff --git a/tools/voluspa/adapters/headerrewrite/forcettl_test.go b/tools/voluspa/adapters/headerrewrite/forcettl_test.go new file mode 100644 index 00000000000..3ae8bacdf63 --- /dev/null +++ b/tools/voluspa/adapters/headerrewrite/forcettl_test.go @@ -0,0 +1,95 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package headerrewrite + +import ( + "errors" + "fmt" + "strings" + "testing" + + "github.com/apache/trafficserver/tools/voluspa" +) + +func TestForceTTL(t *testing.T) { + + plugin := &ForceTTLAdapter{} + + if plugin.Type() != "force_ttl" { + t.Errorf("Type() returned unexpected %q", plugin.Type()) + } + + if fmt.Sprintf("%v", plugin.ConfigParameters()) != "[force_ttl]" { + t.Errorf("ConfigParameters() returned unexpected %v", plugin.ConfigParameters()) + } + + for i, tt := range []struct { + param interface{} + expected string + err error + }{ + { + param: nil, + err: errors.New(`option "force_ttl": value for "force_ttl" is not a string: `), + }, + { + param: true, + err: errors.New(`option "force_ttl": value for "force_ttl" is not a string: true`), + }, + { + param: 42, + err: errors.New(`option "force_ttl": value for "force_ttl" is not a string: 42`), + }, + { + param: "5d", + expected: ` +cond %{READ_RESPONSE_HDR_HOOK} [AND] +cond %{STATUS} >199 [AND] +cond %{STATUS} <400 + set-header Cache-Control "max-age=432000, public"`, + }, + { + param: "no-cache, no-store, must-revalidate, stale-while-revalidate=30", + expected: ` +cond %{READ_RESPONSE_HDR_HOOK} [AND] +cond %{STATUS} >199 [AND] +cond %{STATUS} <400 + set-header Cache-Control "no-cache, no-store, must-revalidate, stale-while-revalidate=30"`, + }, + } { + + ro := &voluspa.RemapOptions{} + env := &voluspa.ConfigEnvironment{RemapOptions: *ro} + + t.Run(fmt.Sprintf("%d-%v", i, tt.param), func(t *testing.T) { + + (*ro)[ForceTTLAdapterParameter] = tt.param + got, err := plugin.Content(env) + if fmt.Sprintf("%v", err) != fmt.Sprintf("%v", tt.err) { + t.Errorf("Content() error expected `%v` got: %v", tt.err, err) + return + } + expected := strings.TrimPrefix(tt.expected, "\n") + if err == nil && got.String() != expected { + t.Errorf("Content() buffer expected:\n%s\ngot:\n%s", expected, got.String()) + } + }) + } +} diff --git a/tools/voluspa/adapters/headerrewrite/header_rewrite.go b/tools/voluspa/adapters/headerrewrite/header_rewrite.go new file mode 100644 index 00000000000..8e77d83540d --- /dev/null +++ b/tools/voluspa/adapters/headerrewrite/header_rewrite.go @@ -0,0 +1,77 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package headerrewrite + +import ( + "bytes" + "fmt" + + "github.com/apache/trafficserver/tools/voluspa" + "github.com/apache/trafficserver/tools/voluspa/adapters/util" +) + +const ( + HeaderRewriteAdapterParameter = "header_rewrite" +) + +func init() { + voluspa.AdaptersRegistry.AddAdapter(&FreeformHeaderRewriteAdapter{}) +} + +type FreeformHeaderRewriteAdapter struct { +} + +func (*FreeformHeaderRewriteAdapter) Weight() int { + return 25 +} + +func (p *FreeformHeaderRewriteAdapter) PluginType() voluspa.PluginType { + return voluspa.CompoundAdapter +} + +func (p *FreeformHeaderRewriteAdapter) Type() voluspa.AdapterType { + return voluspa.AdapterType("raw_header_rewrite") +} + +func (p *FreeformHeaderRewriteAdapter) CompoundType() voluspa.AdapterType { + return adapterType +} + +func (p *FreeformHeaderRewriteAdapter) Name() string { + return "" +} + +func (p *FreeformHeaderRewriteAdapter) ConfigParameters() []string { + return []string{HeaderRewriteAdapterParameter} +} + +func (p *FreeformHeaderRewriteAdapter) SharedLibraryName() string { + return "header_rewrite.so" +} + +func (p *FreeformHeaderRewriteAdapter) Content(env *voluspa.ConfigEnvironment) (*bytes.Buffer, error) { + val, err := env.RemapOptions.ValueByNameAsString(HeaderRewriteAdapterParameter) + if err != nil { + return nil, util.FormatError(HeaderRewriteAdapterParameter, err) + } + + // Do nothing with the config, just pass through + return bytes.NewBufferString(fmt.Sprintf("%s\n", val)), nil +} diff --git a/tools/voluspa/adapters/headerrewrite/log_cookie_header.go b/tools/voluspa/adapters/headerrewrite/log_cookie_header.go new file mode 100644 index 00000000000..14043051c91 --- /dev/null +++ b/tools/voluspa/adapters/headerrewrite/log_cookie_header.go @@ -0,0 +1,107 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package headerrewrite + +import ( + "bytes" + "fmt" + "sort" + + "github.com/apache/trafficserver/tools/voluspa" +) + +const ( + logCookieDirective = "log_cookie" + logHeaderDirective = "log_header" +) + +func init() { + voluspa.AdaptersRegistry.AddAdapter(&LogAdapter{}) +} + +type LogAdapter struct { +} + +func (p *LogAdapter) PluginType() voluspa.PluginType { + return voluspa.CompoundAdapter +} + +func (p *LogAdapter) Type() voluspa.AdapterType { + return voluspa.AdapterType("log_field") +} + +func (p *LogAdapter) CompoundType() voluspa.AdapterType { + return adapterType +} + +func (p *LogAdapter) ConfigParameters() []string { + return []string{logCookieDirective, logHeaderDirective} +} + +func (p *LogAdapter) logCookieHeader(logFields map[string]string, content *bytes.Buffer, parameterName string, index int) (*bytes.Buffer, int) { + var logFieldNames []string + + for k := range logFields { + logFieldNames = append(logFieldNames, k) + } + sort.Strings(logFieldNames) + + content.WriteString(fmt.Sprintf( + `cond %%{REMAP_PSEUDO_HOOK}`, + )) + content.WriteString("\n") + + for _, logFieldName := range logFieldNames { + logFieldValue := logFields[logFieldName] + if parameterName == "log_cookie" { + content.WriteString(fmt.Sprintf( + ` set-header Bazinga%d %%{COOKIE:%s}`, + index, logFieldValue, + )) + index++ + } else if parameterName == "log_header" { + content.WriteString(fmt.Sprintf( + ` set-header Bazinga%d %%{HEADER:%s}`, + index, logFieldValue, + )) + index++ + } + content.WriteString("\n") + } + return content, index +} + +func (p *LogAdapter) Content(env *voluspa.ConfigEnvironment) (*bytes.Buffer, error) { + content := &bytes.Buffer{} + index := 1 + + // Log Cookies + logFields, _ := env.RemapOptions.ValueByNameAsStringMapString(logCookieDirective) + if len(logFields) > 0 { + content, index = p.logCookieHeader(logFields, content, logCookieDirective, index) + } + + // Log headers + logFields, _ = env.RemapOptions.ValueByNameAsStringMapString(logHeaderDirective) + if len(logFields) > 0 { + content, _ = p.logCookieHeader(logFields, content, logHeaderDirective, index) + } + return content, nil +} diff --git a/tools/voluspa/adapters/headerrewrite/log_type.go b/tools/voluspa/adapters/headerrewrite/log_type.go new file mode 100644 index 00000000000..caa085e64af --- /dev/null +++ b/tools/voluspa/adapters/headerrewrite/log_type.go @@ -0,0 +1,74 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package headerrewrite + +import ( + "bytes" + + "github.com/apache/trafficserver/tools/voluspa" + "github.com/apache/trafficserver/tools/voluspa/adapters/util" +) + +const ( + LogTypeAdapterParameter = "log_type" +) + +func init() { + voluspa.AdaptersRegistry.AddAdapter(&LogTypeAdapter{}) +} + +type LogTypeAdapter struct { +} + +func (p *LogTypeAdapter) PluginType() voluspa.PluginType { + return voluspa.CompoundAdapter +} + +func (p *LogTypeAdapter) Type() voluspa.AdapterType { + return voluspa.AdapterType("log_type") +} + +func (p *LogTypeAdapter) CompoundType() voluspa.AdapterType { + return adapterType +} + +func (p *LogTypeAdapter) Name() string { + return "" +} + +func (p *LogTypeAdapter) ConfigParameters() []string { + return []string{LogTypeAdapterParameter} +} + +func (p *LogTypeAdapter) Content(env *voluspa.ConfigEnvironment) (*bytes.Buffer, error) { + content := &bytes.Buffer{} + + val, err := env.RemapOptions.ValueByNameAsString(LogTypeAdapterParameter) + if err != nil { + return nil, err + } + + if val == "public" { + return nil, nil + } + content.WriteString(util.FormatSimpleHeaderRewrite("REMAP_PSEUDO_HOOK", `set-header @cdnlog "private"`)) + + return content, nil +} diff --git a/tools/voluspa/adapters/headerrewrite/proxy_cache_control.go b/tools/voluspa/adapters/headerrewrite/proxy_cache_control.go new file mode 100644 index 00000000000..9d2a36b48a0 --- /dev/null +++ b/tools/voluspa/adapters/headerrewrite/proxy_cache_control.go @@ -0,0 +1,77 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package headerrewrite + +import ( + "bytes" + "fmt" + + "github.com/apache/trafficserver/tools/voluspa" + "github.com/apache/trafficserver/tools/voluspa/adapters/util" +) + +const ( + ProxyCacheControlAdapterParameter = "proxy_cache_control" +) + +func init() { + voluspa.AdaptersRegistry.AddAdapter(&ProxyCacheControlAdapter{}) +} + +type ProxyCacheControlAdapter struct { +} + +func (p *ProxyCacheControlAdapter) PluginType() voluspa.PluginType { + return voluspa.CompoundAdapter +} + +func (p *ProxyCacheControlAdapter) Type() voluspa.AdapterType { + return voluspa.AdapterType("proxy_cache_control") +} + +func (p *ProxyCacheControlAdapter) CompoundType() voluspa.AdapterType { + return adapterType +} + +func (p *ProxyCacheControlAdapter) ConfigParameters() []string { + return []string{ProxyCacheControlAdapterParameter} +} + +func (p *ProxyCacheControlAdapter) Content(env *voluspa.ConfigEnvironment) (*bytes.Buffer, error) { + header, err := env.RemapOptions.ValueByNameAsString(ProxyCacheControlAdapterParameter) + if err != nil { + return nil, util.FormatError(ProxyCacheControlAdapterParameter, err) + } + + content := &bytes.Buffer{} + + content.WriteString(fmt.Sprintf( + `cond %%{READ_RESPONSE_HDR_HOOK} + set-header @Original-Cache-Control %%{HEADER:Cache-Control} + set-header Cache-Control %%{HEADER:%s} + +cond %%{SEND_RESPONSE_HDR_HOOK} + set-header Cache-Control %%{HEADER:@Original-Cache-Control} + rm-header Proxy-Cache-Control`, + header, + )) + + return content, nil +} diff --git a/tools/voluspa/adapters/headerrewrite/receipt.go b/tools/voluspa/adapters/headerrewrite/receipt.go new file mode 100644 index 00000000000..e077132bff4 --- /dev/null +++ b/tools/voluspa/adapters/headerrewrite/receipt.go @@ -0,0 +1,101 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package headerrewrite + +import ( + "bytes" + "fmt" + + "github.com/apache/trafficserver/tools/voluspa" + "github.com/apache/trafficserver/tools/voluspa/adapters/util" +) + +const ( + ReceiptsAdapterParameter = "receipts" + ReceiptsAdapterParameterEnabled = "enabled" + ReceiptsAdapterParameterName = "name" +) + +func init() { + voluspa.AdaptersRegistry.AddAdapter(&ReceiptAdapter{}) +} + +type ReceiptAdapter struct { +} + +func (*ReceiptAdapter) Weight() int { + return 25 +} + +func (p *ReceiptAdapter) PluginType() voluspa.PluginType { + return voluspa.CompoundAdapter +} + +func (p *ReceiptAdapter) Type() voluspa.AdapterType { + return voluspa.Receipt +} + +func (p *ReceiptAdapter) CompoundType() voluspa.AdapterType { + return adapterType +} + +func (p *ReceiptAdapter) Name() string { + return "" +} + +func (p *ReceiptAdapter) ConfigParameters() []string { + return []string{ReceiptsAdapterParameter} +} + +func (p *ReceiptAdapter) Content(env *voluspa.ConfigEnvironment) (*bytes.Buffer, error) { + content := &bytes.Buffer{} + + receiptsConfig, err := env.RemapOptions.ValueByNameAsStringMapInterface(ReceiptsAdapterParameter) + if err != nil { + return nil, err + } + + value, hasOption := receiptsConfig[ReceiptsAdapterParameterEnabled] + if hasOption { + bvalue, ok := value.(bool) + if !ok { + return nil, fmt.Errorf("invalid 'enabled' value (%s)", value) + } + + if !bvalue { + return content, nil + } + } + + receiptName := env.Property + value, hasOption = receiptsConfig[ReceiptsAdapterParameterName] + if hasOption { + strvalue, ok := value.(string) + if !ok { + return nil, fmt.Errorf("invalid 'name' value (%s)", value) + } + receiptName = strvalue + } + + receipt := fmt.Sprintf("%s{{hosttype}}", receiptName) + content.WriteString(util.FormatSimpleHeaderRewrite("REMAP_PSEUDO_HOOK", fmt.Sprintf(`set-header @ReceiptService "%s"`, receipt))) + + return content, nil +} diff --git a/tools/voluspa/adapters/headerrewrite/remove_header.go b/tools/voluspa/adapters/headerrewrite/remove_header.go new file mode 100644 index 00000000000..4843137fd16 --- /dev/null +++ b/tools/voluspa/adapters/headerrewrite/remove_header.go @@ -0,0 +1,74 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package headerrewrite + +import ( + "bytes" + "fmt" + + "github.com/apache/trafficserver/tools/voluspa" + "github.com/apache/trafficserver/tools/voluspa/adapters/util" +) + +const ( + RemoveHeaderAdapterParameter = "remove_header" + RemoveHeaderAdapterOriginParameter = "remove_header_origin" +) + +func init() { + voluspa.AdaptersRegistry.AddAdapter(&RemoveHeaderAdapter{}) +} + +type RemoveHeaderAdapter struct { +} + +func (p *RemoveHeaderAdapter) PluginType() voluspa.PluginType { + return voluspa.CompoundAdapter +} + +func (p *RemoveHeaderAdapter) Type() voluspa.AdapterType { + return voluspa.AdapterType("remove_header") +} + +func (p *RemoveHeaderAdapter) CompoundType() voluspa.AdapterType { + return adapterType +} + +func (p *RemoveHeaderAdapter) ConfigParameters() []string { + return []string{RemoveHeaderAdapterParameter, RemoveHeaderAdapterOriginParameter} +} + +func (p *RemoveHeaderAdapter) Content(env *voluspa.ConfigEnvironment) (*bytes.Buffer, error) { + header, _ := env.RemapOptions.ValueByNameAsString(RemoveHeaderAdapterParameter) + + var buf bytes.Buffer + if len(header) > 0 { + buf.WriteString(util.FormatSimpleHeaderRewrite("REMAP_PSEUDO_HOOK", fmt.Sprintf("rm-header %s", header))) + } + header, _ = env.RemapOptions.ValueByNameAsString(RemoveHeaderAdapterOriginParameter) + if len(header) > 0 { + if buf.Len() > 0 { + buf.WriteByte('\n') + } + buf.WriteString(util.FormatSimpleHeaderRewrite("SEND_REQUEST_HDR_HOOK", fmt.Sprintf("rm-header %s", header))) + } + + return &buf, nil +} diff --git a/tools/voluspa/adapters/headerrewrite/rewrite_utils.go b/tools/voluspa/adapters/headerrewrite/rewrite_utils.go new file mode 100644 index 00000000000..6634a62477b --- /dev/null +++ b/tools/voluspa/adapters/headerrewrite/rewrite_utils.go @@ -0,0 +1,23 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package headerrewrite + +const conditionStatusCodeSuccess = `cond %{STATUS} >199 [AND] +cond %{STATUS} <400` diff --git a/tools/voluspa/adapters/headerrewrite/set_header.go b/tools/voluspa/adapters/headerrewrite/set_header.go new file mode 100644 index 00000000000..011036773e0 --- /dev/null +++ b/tools/voluspa/adapters/headerrewrite/set_header.go @@ -0,0 +1,75 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package headerrewrite + +import ( + "bytes" + "fmt" + + "github.com/apache/trafficserver/tools/voluspa" + "github.com/apache/trafficserver/tools/voluspa/adapters/util" +) + +const ( + SetHeaderAdapterParameter = "set_header" + SetHeaderAdapterOriginParameter = "set_header_origin" +) + +func init() { + voluspa.AdaptersRegistry.AddAdapter(&SetHeaderAdapter{}) +} + +type SetHeaderAdapter struct { +} + +func (p *SetHeaderAdapter) PluginType() voluspa.PluginType { + return voluspa.CompoundAdapter +} + +func (p *SetHeaderAdapter) Type() voluspa.AdapterType { + return voluspa.AdapterType("set_header") +} + +func (p *SetHeaderAdapter) CompoundType() voluspa.AdapterType { + return adapterType +} + +func (p *SetHeaderAdapter) ConfigParameters() []string { + return []string{SetHeaderAdapterParameter, SetHeaderAdapterOriginParameter} +} + +func (p *SetHeaderAdapter) Content(env *voluspa.ConfigEnvironment) (*bytes.Buffer, error) { + header, _ := env.RemapOptions.ValueByNameAsString(SetHeaderAdapterParameter) + + var buf bytes.Buffer + if len(header) > 0 { + buf.WriteString(util.FormatSimpleHeaderRewrite("READ_RESPONSE_HDR_HOOK", fmt.Sprintf("set-header %s", header))) + } + + header, _ = env.RemapOptions.ValueByNameAsString(SetHeaderAdapterOriginParameter) + if len(header) > 0 { + if buf.Len() > 0 { + buf.WriteByte('\n') + } + buf.WriteString(util.FormatSimpleHeaderRewrite("SEND_REQUEST_HDR_HOOK", fmt.Sprintf("set-header %s", header))) + } + + return &buf, nil +} diff --git a/tools/voluspa/adapters/init.go b/tools/voluspa/adapters/init.go new file mode 100644 index 00000000000..c83ad48814d --- /dev/null +++ b/tools/voluspa/adapters/init.go @@ -0,0 +1,24 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package adapters + +import _ "github.com/apache/trafficserver/tools/voluspa/adapters/confremap" // for convenience +import _ "github.com/apache/trafficserver/tools/voluspa/adapters/headerrewrite" // for convenience +import _ "github.com/apache/trafficserver/tools/voluspa/adapters/lua" // for convenience diff --git a/tools/voluspa/adapters/lua/base_lua.go b/tools/voluspa/adapters/lua/base_lua.go new file mode 100644 index 00000000000..19fa496a63f --- /dev/null +++ b/tools/voluspa/adapters/lua/base_lua.go @@ -0,0 +1,65 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package lua + +import ( + "bytes" + + "github.com/apache/trafficserver/tools/voluspa" +) + +func init() { + voluspa.AdaptersRegistry.AddAdapter(&BaseLuaAdapter{}) +} + +type BaseLuaAdapter struct { +} + +func (*BaseLuaAdapter) Weight() int { + return 99 +} + +func (p *BaseLuaAdapter) PluginType() voluspa.PluginType { + return voluspa.CompoundAdapter +} + +func (p *BaseLuaAdapter) Type() voluspa.AdapterType { + return adapterType +} + +func (p *BaseLuaAdapter) CompoundType() voluspa.AdapterType { + return adapterType +} + +func (p *BaseLuaAdapter) Name() string { + return luaScriptName +} + +func (p *BaseLuaAdapter) SharedLibraryName() string { + return "tslua.so" +} + +func (p *BaseLuaAdapter) ConfigParameters() []string { + return []string{""} +} + +func (p *BaseLuaAdapter) Content(env *voluspa.ConfigEnvironment) (*bytes.Buffer, error) { + return &bytes.Buffer{}, nil +} diff --git a/tools/voluspa/adapters/lua/common.go b/tools/voluspa/adapters/lua/common.go new file mode 100644 index 00000000000..470aa3f67e4 --- /dev/null +++ b/tools/voluspa/adapters/lua/common.go @@ -0,0 +1,25 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package lua + +import "github.com/apache/trafficserver/tools/voluspa" + +var adapterType = voluspa.AdapterType("lua") +var luaScriptName = "youdidntsetluascriptname.lua" diff --git a/tools/voluspa/adapters/lua/echo_cors.go b/tools/voluspa/adapters/lua/echo_cors.go new file mode 100644 index 00000000000..b5e01e276f7 --- /dev/null +++ b/tools/voluspa/adapters/lua/echo_cors.go @@ -0,0 +1,83 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package lua + +import ( + "bytes" + "fmt" + + "github.com/apache/trafficserver/tools/voluspa" +) + +const ( + EchoCORSAdapterParameter = "echo_cors" +) + +func init() { + voluspa.AdaptersRegistry.AddAdapter(&EchoCORSAdapter{}) + + // Every standard script in adapters/lua has to set this + luaScriptName = "echo_cors.lua" +} + +type EchoCORSAdapter struct { +} + +func (p *EchoCORSAdapter) PluginType() voluspa.PluginType { + return voluspa.CompoundAdapter +} + +func (p *EchoCORSAdapter) Type() voluspa.AdapterType { + return voluspa.AdapterType("echo_cors") +} + +func (p *EchoCORSAdapter) CompoundType() voluspa.AdapterType { + return adapterType +} + +func (p *EchoCORSAdapter) ConfigParameters() []string { + return []string{EchoCORSAdapterParameter} +} + +func (p *EchoCORSAdapter) Content(env *voluspa.ConfigEnvironment) (*bytes.Buffer, error) { + content := &bytes.Buffer{} + + headerName, err := env.RemapOptions.ValueByNameAsString(EchoCORSAdapterParameter) + if err != nil { + return nil, err + } + + content.WriteString(fmt.Sprintf( + ` +function send_response() + ts.client_response.header['Access-Control-Allow-Origin'] = ts.ctx['origin'] + ts.client_response.header['Access-Control-Allow-Authentication'] = 'True' + return 0 +end + +function do_remap() + ts.ctx['origin'] = ts.client_request.header.%s + ts.hook(TS_LUA_HOOK_SEND_RESPONSE_HDR, send_response) + return 0 +end + `, headerName)) + + return content, nil +} diff --git a/tools/voluspa/adapters/lua/lua.go b/tools/voluspa/adapters/lua/lua.go new file mode 100644 index 00000000000..1fcf35c0254 --- /dev/null +++ b/tools/voluspa/adapters/lua/lua.go @@ -0,0 +1,78 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package lua + +import ( + "bytes" + "fmt" + + "github.com/apache/trafficserver/tools/voluspa" + "github.com/apache/trafficserver/tools/voluspa/adapters/util" +) + +const ( + LuaAdapterParameter = "lua" +) + +func init() { + voluspa.AdaptersRegistry.AddAdapter(&FreeformLuaAdapter{}) + luaScriptName = "inline.lua" +} + +type FreeformLuaAdapter struct { +} + +func (*FreeformLuaAdapter) Weight() int { + return 25 +} + +func (p *FreeformLuaAdapter) PluginType() voluspa.PluginType { + return voluspa.CompoundAdapter +} + +func (p *FreeformLuaAdapter) Type() voluspa.AdapterType { + return voluspa.AdapterType("raw_lua") +} + +func (p *FreeformLuaAdapter) CompoundType() voluspa.AdapterType { + return adapterType +} + +func (p *FreeformLuaAdapter) Name() string { + return "" +} + +func (p *FreeformLuaAdapter) ConfigParameters() []string { + return []string{LuaAdapterParameter} +} + +func (p *FreeformLuaAdapter) SharedLibraryName() string { + return "tslua.so" +} + +func (p *FreeformLuaAdapter) Content(env *voluspa.ConfigEnvironment) (*bytes.Buffer, error) { + val, err := env.RemapOptions.ValueByNameAsString(LuaAdapterParameter) + if err != nil { + return nil, util.FormatError(LuaAdapterParameter, err) + } + + // Do nothing with the config, just pass through + return bytes.NewBufferString(fmt.Sprintf("%s\n", val)), nil +} diff --git a/tools/voluspa/adapters/null.go b/tools/voluspa/adapters/null.go new file mode 100644 index 00000000000..898de95cf38 --- /dev/null +++ b/tools/voluspa/adapters/null.go @@ -0,0 +1,51 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package adapters + +import ( + "github.com/apache/trafficserver/tools/voluspa" +) + +func init() { + voluspa.AdaptersRegistry.AddAdapter(&NullAdapter{}) +} + +type NullAdapter struct { +} + +func (p *NullAdapter) PluginType() voluspa.PluginType { + return voluspa.GeneralAdapter +} + +func (p *NullAdapter) Type() voluspa.AdapterType { + return voluspa.AdapterType("null") +} + +func (p *NullAdapter) Name() string { + return "null" +} + +func (p *NullAdapter) ConfigParameters() []string { + return []string{"null"} +} + +func (p *NullAdapter) PParams(env *voluspa.ConfigEnvironment) ([]string, error) { + return nil, nil +} diff --git a/tools/voluspa/adapters/redirect.go b/tools/voluspa/adapters/redirect.go new file mode 100644 index 00000000000..55b0c089e54 --- /dev/null +++ b/tools/voluspa/adapters/redirect.go @@ -0,0 +1,87 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package adapters + +import ( + "bytes" + "fmt" + + "github.com/apache/trafficserver/tools/voluspa" +) + +func init() { + voluspa.AdaptersRegistry.AddAdapter(&RedirectAdapter{}) +} + +type RedirectAdapter struct { +} + +func (*RedirectAdapter) Weight() int { + return 30 +} + +func (p *RedirectAdapter) PluginType() voluspa.PluginType { + return voluspa.GeneralAdapter +} + +func (p *RedirectAdapter) Type() voluspa.AdapterType { + return voluspa.AdapterType("redirect") +} + +func (p *RedirectAdapter) Name() string { + return "redirect" +} + +func (p *RedirectAdapter) SharedLibraryName() string { + return regexRemapSharedLibraryName +} + +func (p *RedirectAdapter) ConfigParameters() []string { + return []string{"redirect"} +} + +func (p *RedirectAdapter) Content(env *voluspa.ConfigEnvironment) (*bytes.Buffer, error) { + content := &bytes.Buffer{} + urls, err := env.RemapOptions.ValueByNameAsStringMapInterface(p.Name()) + if err != nil { + return nil, err + } + + url, ok := urls["url"].(string) + if !ok { + return nil, fmt.Errorf("Expecting url as string and Redirect url is mandatory") + } + + // default status code to 302 if not provided + httpCode, ok := urls["http_code"].(int) + if !ok { + httpCode = 302 + } + + // configurable source regex pattern defaults to (.*) + src, ok := urls["src"] + if !ok { + src = "(.*)" + } + + content.WriteString(fmt.Sprintf("%s %s @status=%d", src, url, httpCode)) + + return content, nil +} diff --git a/tools/voluspa/adapters/regex_remap.go b/tools/voluspa/adapters/regex_remap.go new file mode 100644 index 00000000000..a4dbbb37310 --- /dev/null +++ b/tools/voluspa/adapters/regex_remap.go @@ -0,0 +1,76 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package adapters + +import ( + "bytes" + + "github.com/apache/trafficserver/tools/voluspa" + "github.com/apache/trafficserver/tools/voluspa/adapters/util" +) + +const ( + RegexRemapAdapterParameter = "regex_remap" + regexRemapSharedLibraryName = "regex_remap.so" +) + +func init() { + voluspa.AdaptersRegistry.AddAdapter(&RegexRemapAdapter{}) +} + +type RegexRemapAdapter struct { +} + +func (*RegexRemapAdapter) Weight() int { + return 15 +} + +func (p *RegexRemapAdapter) PluginType() voluspa.PluginType { + return voluspa.GeneralAdapter +} + +func (p *RegexRemapAdapter) Type() voluspa.AdapterType { + return voluspa.AdapterType("regex_remap") +} + +func (p *RegexRemapAdapter) ConfigLocations() []voluspa.ConfigLocation { + return []voluspa.ConfigLocation{voluspa.ParentConfig} +} + +func (p *RegexRemapAdapter) Name() string { + return "regex" +} + +func (p *RegexRemapAdapter) SharedLibraryName() string { + return regexRemapSharedLibraryName +} + +func (p *RegexRemapAdapter) ConfigParameters() []string { + return []string{RegexRemapAdapterParameter} +} + +func (p *RegexRemapAdapter) Content(env *voluspa.ConfigEnvironment) (*bytes.Buffer, error) { + regex, err := env.RemapOptions.ValueByNameAsString(RegexRemapAdapterParameter) + if err != nil { + return nil, util.FormatError(RegexRemapAdapterParameter, err) + } + + return bytes.NewBufferString(regex), nil +} diff --git a/tools/voluspa/adapters/s3_auth.go b/tools/voluspa/adapters/s3_auth.go new file mode 100644 index 00000000000..b9776164c0c --- /dev/null +++ b/tools/voluspa/adapters/s3_auth.go @@ -0,0 +1,139 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package adapters + +import ( + "strconv" + + "fmt" + "strings" + + "github.com/apache/trafficserver/tools/voluspa" +) + +const ( + S3AuthAdapterParameter = "s3" +) + +func init() { + voluspa.AdaptersRegistry.AddAdapter(&S3AuthAdapter{}) +} + +type S3AuthAdapter struct { +} + +func (*S3AuthAdapter) Weight() int { + return 16 +} + +func (p *S3AuthAdapter) PluginType() voluspa.PluginType { + return voluspa.GeneralAdapter +} + +func (p *S3AuthAdapter) Type() voluspa.AdapterType { + return voluspa.AdapterType("s3_auth") +} + +func (p *S3AuthAdapter) SharedLibraryName() string { + return "s3_auth.so" +} + +func (p *S3AuthAdapter) ConfigLocations() []voluspa.ConfigLocation { + return []voluspa.ConfigLocation{voluspa.ParentConfig} +} + +func (p *S3AuthAdapter) ConfigParameters() []string { + return []string{S3AuthAdapterParameter} +} + +func (p *S3AuthAdapter) PParams(env *voluspa.ConfigEnvironment) ([]string, error) { + var alt map[string]interface{} + alt, err := env.RemapOptions.ValueByNameAsStringMapInterface(S3AuthAdapterParameter) + if err != nil { + return nil, err + } + + args, err := p.processArgs(alt) + if err != nil { + return nil, err + } + + return args, nil +} + +func (p *S3AuthAdapter) processArgs(alt map[string]interface{}) ([]string, error) { + var args []string + args = p.maybeAddArgs(alt["path"], "config", args) + args = p.maybeAddArgs(alt["auth"], "config", args) + args = p.maybeAddArgs(alt["virtual_host"], "virtual_host", args) + args = p.maybeAddArgs(alt["version"], "version", args) + args = p.maybeAddArgs(alt["v4_include_headers"], "v4-include-headers", args) + args = p.maybeAddArgs(alt["v4_exclude_headers"], "v4-exclude-headers", args) + args = p.maybeAddArgs(alt["region_map"], "v4-region-map", args) + + // if this is v4 and there is no v4_exclude_headers defined, add a @pparam=--v4-exclude-headers=x-forwarded-for,forwarded,via,authorization + _, excludeHeadersDefined := alt["v4_exclude_headers"] + if alt["version"] == 4 && !excludeHeadersDefined { + args = p.maybeAddArgs("x-forwarded-for,forwarded,via,authorization", "v4-exclude-headers", args) + } + + return args, nil +} + +func (p *S3AuthAdapter) paramToValue(i interface{}) string { + switch val := i.(type) { + case string: + return val + case int: + return strconv.Itoa(val) + case bool: + return strconv.FormatBool(val) + case []string: + return strings.Join(val, ",") + case []interface{}: + var vals []string + for _, v1 := range i.([]interface{}) { + vS := v1.(string) + vals = append(vals, vS) + } + return strings.Join(vals, ",") + default: + return "" + } +} + +func (p *S3AuthAdapter) maybeAddArgs(option interface{}, param string, args []string) []string { + value := p.paramToValue(option) + switch param { + case "virtual_host": + if value == "true" { + return append(args, "--virtual_host") + } + case "config": + if len(value) > 0 { + return append(args, []string{"--config", value}...) + } + default: + if len(value) > 0 { + return append(args, fmt.Sprintf("--%s=%s", param, value)) + } + } + return args +} diff --git a/tools/voluspa/adapters/strip_query.go b/tools/voluspa/adapters/strip_query.go new file mode 100644 index 00000000000..57c9a4d41b9 --- /dev/null +++ b/tools/voluspa/adapters/strip_query.go @@ -0,0 +1,57 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package adapters + +import ( + "bytes" + + "github.com/apache/trafficserver/tools/voluspa" +) + +func init() { + voluspa.AdaptersRegistry.AddAdapter(&StripQueryAdapter{}) +} + +type StripQueryAdapter struct { +} + +func (p *StripQueryAdapter) PluginType() voluspa.PluginType { + return voluspa.GeneralAdapter +} + +func (p *StripQueryAdapter) Type() voluspa.AdapterType { + return voluspa.AdapterType("strip_query") +} + +func (p *StripQueryAdapter) Name() string { + return "strip_query" +} + +func (p *StripQueryAdapter) SharedLibraryName() string { + return regexRemapSharedLibraryName +} + +func (p *StripQueryAdapter) ConfigParameters() []string { + return []string{"strip_query"} +} + +func (p *StripQueryAdapter) Content(env *voluspa.ConfigEnvironment) (*bytes.Buffer, error) { + return bytes.NewBufferString(". $s://$t/$P\n"), nil +} diff --git a/tools/voluspa/adapters/util/ttl_utils.go b/tools/voluspa/adapters/util/ttl_utils.go new file mode 100644 index 00000000000..d18f0a83aa8 --- /dev/null +++ b/tools/voluspa/adapters/util/ttl_utils.go @@ -0,0 +1,62 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package util + +import ( + "fmt" + "regexp" + "strconv" + "time" +) + +var durationRE = regexp.MustCompile(`^(\d+)([a-z])$`) + +// DurationToSeconds takes a duration string and returns number of seconds +// required to be a single time unit (eg 7d or 24h or 3s) +// complex duration not supported(yet) (eg 7d24h3s) +func DurationToSeconds(value string) (string, error) { + if value == "0" { + return value, nil + } + + if !durationRE.MatchString(value) { + return "", fmt.Errorf("Bad duration \"%s\"", value) + } + + matches := durationRE.FindStringSubmatch(value) + count, err := strconv.ParseUint(matches[1], 10, 64) + if err != nil { + return "", fmt.Errorf("Bad duration \"%s\"", value) + } + + switch matches[2] { + case "d": + return strconv.FormatUint(count*60*60*24, 10), nil + case "w": + return strconv.FormatUint(count*60*60*24*7, 10), nil + + default: + duration, err := time.ParseDuration(value) + if err == nil { + return fmt.Sprintf("%d", int(duration.Seconds())), nil + } + return "", fmt.Errorf("Unhandled spec '%s' for '%s'. err=%s", matches[2], value, err) + } +} diff --git a/tools/voluspa/adapters/util/ttl_utils_test.go b/tools/voluspa/adapters/util/ttl_utils_test.go new file mode 100644 index 00000000000..d8baf8f178e --- /dev/null +++ b/tools/voluspa/adapters/util/ttl_utils_test.go @@ -0,0 +1,85 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package util + +import "testing" + +func TestDurationToSeconds(t *testing.T) { + in := "30d" + expected := "2592000" + val, err := DurationToSeconds(in) + + if err != nil || val != expected { + t.Fatalf("Got '%s' expected '%s' err=%s", val, expected, err) + } + + in = "0" + expected = "0" + val, err = DurationToSeconds(in) + + if err != nil || val != expected { + t.Fatalf("Got '%s' expected '%s' err=%s", val, expected, err) + } + + in = "7w" + expected = "4233600" + val, err = DurationToSeconds(in) + + if err != nil || val != expected { + t.Fatalf("Got '%s' expected '%s' err=%s", val, expected, err) + } + + in = "1h" + expected = "3600" + val, err = DurationToSeconds(in) + + if err != nil || val != expected { + t.Fatalf("Got '%s' expected '%s' err=%s", val, expected, err) + } + + in = "2h3s" + expected = "7203" + val, err = DurationToSeconds(in) + + if err == nil || val == expected { + t.Fatalf("Got '%s' expected '%s' err=%s", val, expected, err) + } + + in = "7r" + val, err = DurationToSeconds(in) + + if err == nil { + t.Fatalf("Expected err converting '%s' val=%s", in, val) + } + + in = "5" + val, err = DurationToSeconds(in) + + if err == nil { + t.Fatalf("Expected err converting '%s' val=%s", in, val) + } + + in = "7w2d3h" + val, err = DurationToSeconds(in) + + if err == nil { + t.Fatalf("Expected err converting '%s' val=%s", in, val) + } +} diff --git a/tools/voluspa/adapters/util/util.go b/tools/voluspa/adapters/util/util.go new file mode 100644 index 00000000000..20a21c81459 --- /dev/null +++ b/tools/voluspa/adapters/util/util.go @@ -0,0 +1,46 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package util + +import ( + "fmt" + "strconv" +) + +func FormatSimpleHeaderRewrite(condition, action string) string { + return fmt.Sprintf("cond %%{%s}\n %s", condition, action) +} + +func FormatError(option string, err error) error { + return fmt.Errorf("option %q: %s", option, err.Error()) +} + +func ParamToValue(i interface{}) string { + switch val := i.(type) { + case string: + return val + case int: + return strconv.Itoa(val) + case bool: + return strconv.FormatBool(val) + default: + return "" + } +} diff --git a/tools/voluspa/adapters/video_background_fetch.go b/tools/voluspa/adapters/video_background_fetch.go new file mode 100644 index 00000000000..23ccf69a2eb --- /dev/null +++ b/tools/voluspa/adapters/video_background_fetch.go @@ -0,0 +1,198 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package adapters + +import ( + "fmt" + "strings" + + "github.com/apache/trafficserver/tools/voluspa" + "github.com/apache/trafficserver/tools/voluspa/adapters/util" +) + +func init() { + voluspa.AdaptersRegistry.AddAdapter(&VideoBackgroundFetchAdapter{}) +} + +const ( + VideoBackgroundFetchDirective = "video_background_fetch" + + VideoBackgroundFetchAdapterFrontend = "frontend" + VideoBackgroundFetchAdapterBackend = "backend" + + VideoBackgroundFetchAdapterFetchCount = "fetch_count" + VideoBackgroundFetchAdapterFetchPolicy = "fetch_policy" + VideoBackgroundFetchAdapterFetchMax = "fetch_max" + VideoBackgroundFetchAdapterAPIHeader = "api_header" + VideoBackgroundFetchAdapterFetchPathPattern = "fetch_path_pattern" + VideoBackgroundFetchAdapterReplaceHost = "replace_host" + VideoBackgroundFetchAdapterNameSpace = "name_space" + VideoBackgroundFetchAdapterMetricsPrefix = "metrics_prefix" + VideoBackgroundFetchAdapterExactMatch = "exact_match" + VideoBackgroundFetchAdapterLogName = "log_name" +) + +type VideoBackgroundFetchAdapter struct { +} + +func (p *VideoBackgroundFetchAdapter) PluginType() voluspa.PluginType { + return voluspa.GeneralAdapter +} + +func (p *VideoBackgroundFetchAdapter) Type() voluspa.AdapterType { + return voluspa.AdapterType("video_background_fetch") +} + +func (p *VideoBackgroundFetchAdapter) ConfigLocations() []voluspa.ConfigLocation { + return []voluspa.ConfigLocation{voluspa.ChildConfig} +} + +func (p *VideoBackgroundFetchAdapter) ConfigParameters() []string { + return []string{VideoBackgroundFetchDirective} +} + +func (p *VideoBackgroundFetchAdapter) Name() string { + return "volcano" +} + +func (p *VideoBackgroundFetchAdapter) SharedLibraryName() string { + return "volcano.so" +} + +func (p *VideoBackgroundFetchAdapter) PParams(env *voluspa.ConfigEnvironment) (voluspa.RolePParams, error) { + + alt, err := env.RemapOptions.ValueByNameAsStringMapInterface(VideoBackgroundFetchDirective) + if err != nil { + return nil, util.FormatError(VideoBackgroundFetchDirective, err) + } + + // process default args + args, err := p.processArgs(alt) + if err != nil { + return nil, err + } + + results := voluspa.RolePParams{} + results[voluspa.DefaultRole] = args + + for _, v := range []string{VideoBackgroundFetchAdapterFrontend, VideoBackgroundFetchAdapterBackend} { + sub, found := alt[v] + if !found { + continue + } + + submap, ok := sub.(map[interface{}]interface{}) + if !ok { + continue + } + + convertedMap, err := convertMap(submap) + if err != nil { + return nil, err + } + + args, err := p.processArgs(convertedMap) + if err != nil { + return nil, err + } + + args = append(args, fmt.Sprintf("--front=%t", v == VideoBackgroundFetchAdapterFrontend)) + + // eg roles_video_fetch_backend or roles_video_fetch_frontend + role := fmt.Sprintf("roles_video_fetch_%s", v) + results[role] = args + } + + return results, nil +} + +// convertMap attempts to create a usable submap from the very generic map returned by RemapOptions +func convertMap(in map[interface{}]interface{}) (map[string]interface{}, error) { + results := make(map[string]interface{}) + for k, v := range in { + ck, ok := k.(string) + if !ok { + return nil, fmt.Errorf("key %+q is not string", k) + } + results[ck] = v + } + return results, nil +} + +func (p *VideoBackgroundFetchAdapter) processArgs(alt map[string]interface{}) ([]string, error) { + var args []string + + args = p.maybeAddArgs(alt[VideoBackgroundFetchAdapterFetchPolicy], "fetch-policy", args) + args = p.maybeAddArgs(alt[VideoBackgroundFetchAdapterFetchCount], "fetch-count", args) + args = p.maybeAddArgs(alt[VideoBackgroundFetchAdapterFetchMax], "fetch-max", args) + args = p.maybeAddArgs(alt[VideoBackgroundFetchAdapterAPIHeader], "api-header", args) + args = p.maybeAddArgs(alt[VideoBackgroundFetchAdapterFetchPathPattern], "fetch-path-pattern", args) + args = p.maybeAddArgs(alt[VideoBackgroundFetchAdapterReplaceHost], "replace-host", args) + args = p.maybeAddArgs(alt[VideoBackgroundFetchAdapterNameSpace], "name-space", args) + args = p.maybeAddArgs(alt[VideoBackgroundFetchAdapterMetricsPrefix], "metrics-prefix", args) + args = p.maybeAddArgs(alt[VideoBackgroundFetchAdapterExactMatch], "exact-match", args) + args = p.maybeAddArgs(alt[VideoBackgroundFetchAdapterLogName], "log-name", args) + + // from here on out, it's parsing fetch_path_pattern + if _, found := alt[VideoBackgroundFetchAdapterFetchPathPattern]; !found { + return args, nil + } + + var fetchPatterns []string + switch val := alt[VideoBackgroundFetchAdapterFetchPathPattern].(type) { + case []interface{}: + for _, ipattern := range val { + pattern, ok := ipattern.(string) + if !ok { + return nil, util.FormatError(VideoBackgroundFetchAdapterFetchPathPattern, fmt.Errorf("Expected a string")) + } + fetchPatterns = append(fetchPatterns, pattern) + } + case []string: + fetchPatterns = val + default: + return nil, util.FormatError(VideoBackgroundFetchAdapterFetchPathPattern, fmt.Errorf("Expected an array of strings")) + } + + for _, pattern := range fetchPatterns { + args = p.maybeAddArgs(pattern, "fetch-path-pattern", args) + } + + return args, nil +} + +func isValidFetchPolicy(policy string) bool { + return policy == "simple" || strings.Contains(policy, "lru") +} + +func (p *VideoBackgroundFetchAdapter) maybeAddArgs(option interface{}, param string, args []string) []string { + value := util.ParamToValue(option) + switch param { + case "fetch-policy": + if isValidFetchPolicy(value) { + return append(args, fmt.Sprintf("--%s=%s", param, value)) + } + default: + if len(value) > 0 { + return append(args, fmt.Sprintf("--%s=%s", param, value)) + } + } + return args +} diff --git a/tools/voluspa/adapters/video_background_fetch_test.go b/tools/voluspa/adapters/video_background_fetch_test.go new file mode 100644 index 00000000000..b9ecb68b04b --- /dev/null +++ b/tools/voluspa/adapters/video_background_fetch_test.go @@ -0,0 +1,103 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package adapters + +import ( + "sort" + "testing" + + "github.com/apache/trafficserver/tools/voluspa" +) + +func TestVideoBackgroundFetch(t *testing.T) { + var plugin VideoBackgroundFetchAdapter + + alt := make(map[interface{}]interface{}) + ro := voluspa.RemapOptions{} + env := &voluspa.ConfigEnvironment{ + Property: "testVideoBackgroundFetch", + RemapOptions: ro, + } + + alt[VideoBackgroundFetchAdapterFetchCount] = "100" + alt[VideoBackgroundFetchAdapterFetchMax] = "1000" + alt[VideoBackgroundFetchAdapterFetchPolicy] = "simple" + alt[VideoBackgroundFetchAdapterAPIHeader] = "header1" + alt[VideoBackgroundFetchAdapterFetchPathPattern] = []string{"(.*)"} + alt[VideoBackgroundFetchAdapterReplaceHost] = "host1" + alt[VideoBackgroundFetchAdapterNameSpace] = "ns1" + alt[VideoBackgroundFetchAdapterMetricsPrefix] = "prefix1" + alt[VideoBackgroundFetchAdapterExactMatch] = "true" + alt[VideoBackgroundFetchAdapterLogName] = "log1" + + ro["video_background_fetch"] = alt + + pparams, err := plugin.PParams(env) + if err != nil { + t.Fatalf("Failed to generate output for plugin. error=%v", err) + } + + out := pparams[voluspa.DefaultRole] + sort.Sort(sort.StringSlice(out)) + + if len(out) == 0 { + t.Fatalf("Failed to generate output for plugin. no output.") + } + + if expected := "--api-header=header1"; out[0] != expected { + t.Fatalf("Error: got '%s'; expected '%s'", out[0], expected) + } + + if expected := "--exact-match=true"; out[1] != expected { + t.Fatalf("Error: got '%s'; expected '%s'", out[1], expected) + } + + if expected := "--fetch-count=100"; out[2] != expected { + t.Fatalf("Error: got '%s'; expected '%s'", out[2], expected) + } + + if expected := "--fetch-max=1000"; out[3] != expected { + t.Fatalf("Error: got '%s'; expected '%s'", out[3], expected) + } + + if expected := "--fetch-path-pattern=(.*)"; out[4] != expected { + t.Fatalf("Error: got '%s'; expected '%s'", out[4], expected) + } + + if expected := "--fetch-policy=simple"; out[5] != expected { + t.Fatalf("Error: got '%s'; expected '%s'", out[5], expected) + } + + if expected := "--log-name=log1"; out[6] != expected { + t.Fatalf("Error: got '%s'; expected '%s'", out[6], expected) + } + + if expected := "--metrics-prefix=prefix1"; out[7] != expected { + t.Fatalf("Error: got '%s'; expected '%s'", out[7], expected) + } + + if expected := "--name-space=ns1"; out[8] != expected { + t.Fatalf("Error: got '%s'; expected '%s'", out[8], expected) + } + + if expected := "--replace-host=host1"; out[9] != expected { + t.Fatalf("Error: got '%s'; expected '%s'", out[9], expected) + } +} diff --git a/tools/voluspa/attributes.go b/tools/voluspa/attributes.go new file mode 100644 index 00000000000..826fec24258 --- /dev/null +++ b/tools/voluspa/attributes.go @@ -0,0 +1,56 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package voluspa + +import "sort" + +// GetCDNs returns a list of CDNs in current config set +func (v *Voluspa) GetCDNs() []string { + cdns := make(map[string]interface{}) + for _, parsedConfig := range v.parsedConfigs { + for cdn := range parsedConfig.cdn { + cdns[cdn] = nil + } + } + + var all []string + for k := range cdns { + all = append(all, k) + } + sort.Strings(all) + + return all +} + +// GetRoles returns a list of Roles in current config set +func (v *Voluspa) GetRoles() []string { + roles := make(map[string]interface{}) + for _, parsedConfig := range v.parsedConfigs { + roles[parsedConfig.role] = nil + } + + var all []string + for k := range roles { + all = append(all, k) + } + sort.Strings(all) + + return all +} diff --git a/tools/voluspa/cdncheck.go b/tools/voluspa/cdncheck.go new file mode 100644 index 00000000000..2501d03b35f --- /dev/null +++ b/tools/voluspa/cdncheck.go @@ -0,0 +1,81 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package voluspa + +import ( + "fmt" +) + +type CDNTest struct { + Type CheckType `json:"type,omitempty" yaml:"type,omitempty"` + DomainNames []string `json:"domain_names,omitempty" yaml:"domain_names,omitempty"` + URLs []string `json:"urls,omitempty" yaml:"urls,omitempty"` + HostTypes []HostType `json:"host_types,omitempty" yaml:"host_types,omitempty"` + Description string `json:"description,omitempty" yaml:"description,omitempty"` + Role string `json:"role,omitempty" yaml:"role,omitempty"` + Range string `json:"range,omitempty" yaml:"range,omitempty"` + Headers map[string]string `json:"headers,omitempty" yaml:"headers,omitempty"` + PurgeBefore bool `json:"purge_before,omitempty" yaml:"purge_before,omitempty"` + Insecure bool `json:"insecure" yaml:"insecure"` + Success *CDNCheckSuccessCriteria `json:"success,omitempty" yaml:"success,omitempty"` + Ciphers []string `json:"ciphers,omitempty" yaml:"ciphers,omitempty"` +} + +type CDNCheckSuccessCriteria struct { + StatusCode int `json:"status_code" yaml:"status_code"` + Headers map[string]string `json:"headers,omitempty" yaml:"headers,omitempty"` + SerialNumbers []string `json:"serial_numbers,omitempty" yaml:"serial_numbers,omitempty"` +} + +type HostType string + +const ( + UnknownHostType HostType = "unknown" + ParentHost HostType = Parent + ChildHost HostType = Child +) + +type CheckType string + +const ( + UnknownCheckType CheckType = "unknown" + HTTPCheckType CheckType = "http" + SSLCheckType CheckType = "ssl" +) + +func (h *HostType) UnmarshalYAML(unmarshal func(interface{}) error) error { + var in string + if err := unmarshal(&in); err != nil { + return err + } + + if len(in) == 0 { + return fmt.Errorf("empty host type") + } + + switch in { + case Parent, Child: + *h = HostType(in) + default: + return fmt.Errorf("unknown host type '%s'", in) + } + + return nil +} diff --git a/tools/voluspa/cmd/voluspa/main.go b/tools/voluspa/cmd/voluspa/main.go new file mode 100644 index 00000000000..ad047e38966 --- /dev/null +++ b/tools/voluspa/cmd/voluspa/main.go @@ -0,0 +1,172 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package main + +import ( + "flag" + "fmt" + "log" + "os" + "sort" + "strings" + + "github.com/apache/trafficserver/tools/voluspa" + _ "github.com/apache/trafficserver/tools/voluspa/adapters" +) + +var ( + configDest = flag.String("dest", "out", "destination directory for written out configs") + installLocation = flag.String("install", voluspa.DefaultTrafficserverConfigurationDir, "trafficserver configuration location") + defaultsLocation = flag.String("defaults", "", "location of default configurations") + schemaLocation = flag.String("schema-location", "", "location of schema (defaults to cwd)") + disableAdapters = flag.String("disable-adapters", "", "a comma separated list of adapters to disable") + + cdnList = flag.String("cdns", "", "a comma separated list of CDNs") + roleList = flag.String("roles", "", "a comma separated list of roles") + showVersion = flag.Bool("version", false, "Show version") + verbose = flag.Bool("verbose", false, "Verbose output") + skipSchemaValidation = flag.Bool("skip-schema-validation", false, "skip schema validation") + treatRolesAsCDN = flag.Bool("promote-roles-to-cdn", false, "promote roles to CDN") + ramDiskRole = flag.String("ramdisk-role", "roles_trafficserver_ramdisk", "The role that indicates this host has a ramdisk volume") + + skipValidation = flag.Bool("skip-validation", false, "skip validation") + validationOnly = flag.Bool("validate-only", false, "stop after validation") + strictMode = flag.Bool("strict", false, "enable strict validation") + dumpSchemaVersion = flag.Int("dump-schema", -1, "dump this version of the schema (0 = highest) to STDOUT and exit 0") +) + +func init() { + log.SetFlags(0) + + flag.Parse() +} + +var exit = func(code int) { + os.Exit(code) +} + +func main() { + if *showVersion { + fmt.Println("voluspa", voluspa.Version) + exit(0) + } + + v, err := voluspa.NewVoluspaWithOptions(&voluspa.Options{ + Verbose: *verbose, + Destination: *configDest, + SkipSchemaValidation: *skipSchemaValidation, + PromoteRolesToCDN: *treatRolesAsCDN, + DefaultsLocation: *defaultsLocation, + SchemaLocation: *schemaLocation, + RamDiskRole: *ramDiskRole, + }, *installLocation) + if err != nil { + log.Printf("%s", err) + exit(1) + } + + if *dumpSchemaVersion >= 0 { + schema, err := v.SchemaDefinition(*dumpSchemaVersion) + if err != nil { + log.Printf("%s", err) + exit(1) + } + fmt.Fprint(os.Stdout, string(schema)) + exit(0) + } + + if len(*disableAdapters) > 0 { + adapters := strings.Split(*disableAdapters, ",") + for _, adapter := range adapters { + if err = voluspa.AdaptersRegistry.RemoveAdapterByType(adapter); err != nil { + log.Fatalf("Could not disable adapter %q: %s", adapter, err) + } + } + } + + if len(flag.Args()) == 0 { + log.Printf("Must specify at least 1 config file") + exit(1) + } + + filenames := flag.Args() + sort.Strings(filenames) + + for _, filename := range filenames { + err = v.AddConfig(filename) + if err != nil { + log.Printf("%s", err) + exit(1) + } + } + + if !*skipValidation { + if err = v.Validate(*strictMode); err == nil { + goto validationDone + } + + fmt.Fprintf(os.Stderr, "voluspa: errors validating configs:\n") + + errs, ok := err.(voluspa.Errors) + if !ok { + fmt.Fprintln(os.Stderr, err) + exit(1) + } + + // sort and unique-ify the set of errors + // like errs.Error() but also uniques and has different format + seen := make(map[string]interface{}) + var errOut []string + for _, e := range errs { + if _, found := seen[e.Error()]; found { + continue + } + errOut = append(errOut, fmt.Sprintf(" - %s", e.Error())) + seen[e.Error()] = nil + } + + sort.Strings(errOut) + + for _, str := range errOut { + fmt.Fprintln(os.Stderr, str) + } + exit(1) + } + +validationDone: + if *validationOnly { + exit(0) + } + + var cdns []string + if len(*cdnList) > 0 { + cdns = strings.Split(*cdnList, ",") + } + + var roles []string + if len(*roleList) > 0 { + roles = strings.Split(*roleList, ",") + } + + if err = v.WriteAllFiles(cdns, roles); err != nil { + log.Printf("%s", err) + exit(1) + } +} diff --git a/tools/voluspa/cmd/voluspa/main_test.go b/tools/voluspa/cmd/voluspa/main_test.go new file mode 100644 index 00000000000..7093562c366 --- /dev/null +++ b/tools/voluspa/cmd/voluspa/main_test.go @@ -0,0 +1,33 @@ +// +build testrunmain + +package main + +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import ( + "testing" +) + +func TestRunMain(t *testing.T) { + exit = func(code int) { + // ignore exit code + } + main() +} diff --git a/tools/voluspa/cmd/voluspa/no_test.go b/tools/voluspa/cmd/voluspa/no_test.go new file mode 100644 index 00000000000..822b16bcbd4 --- /dev/null +++ b/tools/voluspa/cmd/voluspa/no_test.go @@ -0,0 +1,20 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package main diff --git a/tools/voluspa/config_generator.go b/tools/voluspa/config_generator.go new file mode 100644 index 00000000000..18c62c1d6eb --- /dev/null +++ b/tools/voluspa/config_generator.go @@ -0,0 +1,567 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package voluspa + +import ( + "bytes" + "fmt" + "net/url" + "sort" + "strings" +) + +// NewCustomerConfig creates a new CustomerConfig from a PropertyConfig +func NewCustomerConfig(p *PropertyConfig, configFilename string, options Options) (*CustomerConfig, error) { + config, err := newCustomerConfig(p) + if err != nil { + return nil, err + } + + if options.PromoteRolesToCDN && isPromotableRole(p.Role) { + parts := strings.SplitN(p.Role, "_", 2) + if len(parts) == 2 { + // If the only CDN is DefaultCDN, purge and use the role + if len(p.CDN) == 1 && p.CDN[0] == DefaultCDN { + config.cdn = make(map[string]interface{}) + } + + config.cdn[parts[1]] = nil + config.role = "" + } + } + + config.filename = configFilename + + configLocations := []ConfigLocation{UnknownLocation} + if p.ParentChild { + configLocations = []ConfigLocation{ChildConfig, ParentConfig} + } + + var mappings []*Mapping + + // mappings are like explicit, but smarter + // They're parent/child aware, no longer requiring different rulesets for each type. + for _, m := range p.Mappings { + + schemes := p.Schemes + if len(m.Schemes) > 0 { + schemes = m.schemes() + } + + for _, alias := range m.Alias { + for _, scheme := range schemes { + for _, configLocation := range configLocations { + + incoming := formatURL(scheme, alias) + originURL := formatURL(scheme, m.Origin) + + // for parent/child, use the first incoming alias as the origin URL. on the parent side, + // we map it as the inbound + if configLocation == ChildConfig && m.RemapOptions.isParentChildEnabled() { + originURL = formatURL(scheme, m.Alias[0]) + } + + if configLocation == ParentConfig && m.RemapOptions.isParentChildEnabled() { + if len(m.Alias) > 0 && alias != m.Alias[0] { + continue + } + } + + // clone separate versions of RemapOptions with the appropriate key set + ro := m.RemapOptions.Clone() + + // clear out manually set attributes + delete(ro, Parent) + delete(ro, Child) + + switch configLocation { + case ParentConfig: + ro[Parent] = true + case ChildConfig: + ro[Child] = true + } + + mapping := &Mapping{ + Origin: originURL, + Alias: []string{incoming}, + Group: DefaultGroup, + Schemes: []string{scheme}, + RegexMap: m.RegexMap, + RuleName: m.RuleName, + RemapOptions: ro, + id: m.id, + ParentOverride: m.ParentOverride, + } + + if configLocation == ChildConfig && m.RemapOptions.isParentChildEnabled() { + // only include the first alias of an alias set. as seen above, we only map + // through the first alias to the parent + if alias == m.Alias[0] { + if len(m.ParentDestDomain) > 0 { + mapping.ParentDestDomain = m.ParentDestDomain + } else { + mapping.ParentDestDomain = stripScheme(incoming) + } + } + } + + mappings = append(mappings, mapping) + } + } + } + } + + for _, m := range p.ExplicitMappings { + m.explicit = true + mappings = append(mappings, m) + } + + for _, m := range mappings { + + schemes := p.Schemes + if len(m.Schemes) > 0 { + schemes = m.schemes() + } + + for _, scheme := range schemes { + for _, incoming := range m.Alias { + + for _, configLocation := range configLocations { + if !forCurrentMatch(configLocation, &m.RemapOptions) { + continue + } + + env := newConfigEnvironment(p, options, m.id, configLocation) + + mrules, err := convertToMappingRules(m.RemapOptions, env, m.explicit) + if err != nil { + return nil, err + } + + if configLocation == ChildConfig && m.RemapOptions.isParentChildEnabled() { + config.parentDomains = append(config.parentDomains, m.ParentDomain()) + } + + incomingURL := formatURL(scheme, incoming) + originURL := formatURL(scheme, m.Origin) + + remap := Remap{ + IncomingURL: incomingURL, + OriginURL: originURL, + Group: m.Group, + Role: m.Role, + RegexMap: m.RegexMap, + mappingRules: mrules, + ConfigLocation: env.ConfigLocation, + sourceConfig: env, + scheme: scheme, + parentOverrides: m.ParentOverride, + parentDomain: m.ParentDomain(), + } + + config.Remaps = append(config.Remaps, remap) + } + } + } + } + + // For now, sort if the config has mappings. Later, introduce the change for explicit and others + if len(p.Mappings) > 0 { + sort.Stable(sort.Reverse(byIncomingURL(config.Remaps))) + } + + return config, nil +} + +func stripScheme(in string) string { + if !strings.Contains(in, "/") { + return in + } + parts := strings.Split(in, "/") + return parts[2] +} + +func forCurrentMatch(configLocation ConfigLocation, ro *RemapOptions) bool { + if configLocation == ChildConfig && ro.isParentConfig() { + return false + } + if configLocation == ParentConfig && ro.isChildConfig() { + return false + } + return true +} + +func (m *Mapping) schemes() []string { + if len(m.Schemes) > 0 { + return m.Schemes + } + + return []string{""} +} + +// formatURL will create URL from scheme and incoming, unless incoming already has scheme applied to it +func formatURL(scheme, incoming string) string { + if strings.HasPrefix(incoming, "/") { + return incoming + } + + if len(scheme) == 0 { + return incoming + } + + u, err := url.Parse(incoming) + if err == nil && strings.HasPrefix(u.Scheme, "http") { + return incoming + } + + // in the error case, it could be a regex URL that url.Parse barfs on. try a brute force method + if strings.HasPrefix(incoming, "http:") || strings.HasPrefix(incoming, "https:") { + return incoming + } + + return fmt.Sprintf("%s://%s", scheme, incoming) +} + +// isParentChildEnabled returns the value specified by "parent_child". defaults to true +func (options *RemapOptions) isParentChildEnabled() bool { + value, err := options.ValueByNameAsBool("parent_child") + if err == nil { + return value + } + return true +} + +// isParentConfig returns the value specified by "parent". defaults to false +func (options *RemapOptions) isParentConfig() bool { + value, err := options.ValueByNameAsBool(Parent) + if err == nil { + return value + } + return false +} + +// isChildConfig returns the value specified by "child". defaults to false +func (options *RemapOptions) isChildConfig() bool { + value, err := options.ValueByNameAsBool(Child) + if err == nil { + return value + } + return false +} + +// Iterate over all "rules". adding to array of *ATSPLugin +// If a general adapter, use general logic +// If an action, use action logic +// otherwise, assume a compound adapter +// - group by type +// Iterate over groups of compounds (calculated above) +// Append to adapter array +// Sort array by weights and return + +func convertToMappingRules(ro RemapOptions, env *ConfigEnvironment, explicit bool) ([]mappingRule, error) { + var adapters []mappingRule + + compounds := make(map[AdapterType][]AdapterType) + + // stable ordering + var options []string + for option := range ro { + options = append(options, option) + } + sort.Strings(options) + + var errs Errors + + seen := make(map[Adapter]interface{}) + for _, option := range options { + if option == "storage_volume" { // prevent warning on rule property that isn't associated with an adapter + continue + } + adapterType := AdaptersRegistry.adapterTypeByConfigName(option) + if adapterType == UnknownAdapter { + // Warn when there's garbage in + if !IsValidConfigurationOption(option) { + errs = append(errs, fmt.Errorf("unknown configuration option %q", option)) + } + continue + } + + vp := AdaptersRegistry.adapterForType(adapterType) + if vp == nil { + errs = append(errs, fmt.Errorf("unknown adapter %q", adapterType)) + continue + } + + // NOTE: even though an adapter can have multiple options associated with it, + // it's the first invocation that only matters. Once inside PParams, the adapter + // is responsible for determining any other usage of itself + if _, ok := seen[vp]; ok { + continue + } + seen[vp] = nil + + if env.ConfigLocation != UnknownLocation { + vpcp, ok := vp.(ParentChildAdapter) + matches := false + if ok { + for _, loc := range vpcp.ConfigLocations() { + if loc == env.ConfigLocation { + matches = true + } + } + + if !matches && explicit { + // ignore and let through + } else if !matches { + continue + } + } + } + + env.RemapOptions = ro + + switch vp.PluginType() { + case GeneralAdapter: + adapter, err := createAdapter(vp, env) + if err != nil { + errs = append(errs, err) + continue + } + + if adapter.hasContent() { + adapters = append(adapters, *adapter) + } + case ActionAdapter: + adapter, err := createAction(vp, env) + if err != nil { + errs = append(errs, err) + continue + } + if adapter.hasContent() { + adapters = append(adapters, *adapter) + } + case CompoundAdapter: + cta, ok := vp.(CompoundTypeAdapter) + if !ok { + errs = append(errs, fmt.Errorf("compound adapter does not implement CompoundTypeAdapter")) + continue + } + compounds[cta.CompoundType()] = append(compounds[cta.CompoundType()], adapterType) + } + } + + for pt, sp := range compounds { + + sort.Sort(receiptsFirst(sp)) + + adapter, err := createCompoundMappingRule(pt, ro, sp, env) + if err != nil { + errs = append(errs, err) + continue + } + + if adapter.hasContent() { + adapters = append(adapters, *adapter) + } + + } + + sort.Stable(byRuleWeight(adapters)) + + if len(errs) > 0 { + return nil, errs + } + + return adapters, nil +} + +func createCompoundMappingRule(compoundType AdapterType, ro RemapOptions, adapters []AdapterType, env *ConfigEnvironment) (*mappingRule, error) { + vp := AdaptersRegistry.adapterForType(compoundType) + if vp == nil { + return nil, fmt.Errorf("unknown AdapterType %q", compoundType) + } + + var errs Errors + var content bytes.Buffer + + adapter := baseCreateAdapter(vp, env) + + ccontent, err := ConfigContent(vp, env) + if err != nil { + errs = append(errs, err) + } + adapter.ConfigContent = ccontent + + ccontent, err = SubConfigContent(vp, env) + if err != nil { + errs = append(errs, err) + } + + if ccontent != nil { + content.Write(ccontent.Bytes()) + } + + var buffer bytes.Buffer + for _, adapterType := range adapters { + vps := AdaptersRegistry.adaptersForType(adapterType) + for _, vp = range vps { + if vp == nil { + return nil, fmt.Errorf("unknown AdapterType %s", adapterType) + } + + // filter out unrelated types + cta, ok := vp.(CompoundTypeAdapter) + if ok && cta.CompoundType() != compoundType { + continue + } + + vpp, ok := vp.(ParameterAdapter) + if ok { + pparams, err := vpp.PParams(env) + if err != nil { + errs = append(errs, err) + } + + for _, val := range pparams { + fmt.Fprintf(&buffer, " @pparam=%s", val) + } + } + + env.RemapOptions = ro + val, err := SubConfigContent(vp, env) + if err != nil { + errs = append(errs, err) + } + + if val != nil && val.Len() > 0 { + content.Write(val.Bytes()) + content.WriteString("\n") + } + } + } + + if len(errs) > 0 { + return nil, errs + } + + if buffer.Len() > 0 { + adapter.ConfigContent.WriteString(buffer.String()) + } else if content.Len() == 0 { + // if no subconfig content, then there's nothing to be output (at least at this time) + adapter.ConfigContent = &bytes.Buffer{} + } + + adapter.SubConfigContent = &content + + return adapter, nil +} + +func baseCreateAdapter(vp Adapter, env *ConfigEnvironment) *mappingRule { + adapter := newMappingRule(vp) + + scp, ok := vp.(SubConfigAdapter) + if ok { + adapter.SubConfigFileName = SubConfigFilename(scp, env) + } + + return adapter +} + +func createAdapter(vp Adapter, env *ConfigEnvironment) (*mappingRule, error) { + adapter := baseCreateAdapter(vp, env) + + content, err := ConfigContent(vp, env) + if err != nil { + return nil, err + } + adapter.ConfigContent = content + + content, err = SubConfigContent(vp, env) + if err != nil { + return nil, err + } + adapter.SubConfigContent = content + + return adapter, nil +} + +func createAction(vp Adapter, env *ConfigEnvironment) (*mappingRule, error) { + adapter := baseCreateAdapter(vp, env) + content, err := SubConfigContent(vp, env) + if err != nil { + return nil, err + } + adapter.SubConfigContent = content + adapter.ConfigContent = content + + return adapter, nil +} + +// receiptsFirst implements sort.Interface, sorting by AdapterType, bubbling Receipt to the top. +type receiptsFirst []AdapterType + +func (s receiptsFirst) Len() int { return len(s) } +func (s receiptsFirst) Swap(i, j int) { s[i], s[j] = s[j], s[i] } +func (s receiptsFirst) Less(i, j int) bool { return s[i] == Receipt } + +// byRuleWeight implements sort.Interface, sorting by a Rule's Weight. +type byRuleWeight []mappingRule + +func (s byRuleWeight) Len() int { return len(s) } +func (s byRuleWeight) Swap(i, j int) { s[i], s[j] = s[j], s[i] } +func (s byRuleWeight) Less(i, j int) bool { return s[i].Weight < s[j].Weight } + +// byIncomingURL will sort by incoming URL domain +// for the same scheme and inbound host: +// http will be put before https +// empty paths come last +// sorting alphabetically +// ignoring host-less and regex remaps +type byIncomingURL []Remap + +func (s byIncomingURL) Len() int { return len(s) } +func (s byIncomingURL) Swap(i, j int) { s[i], s[j] = s[j], s[i] } +func (s byIncomingURL) Less(i, j int) bool { + if s[i].RegexMap || s[j].RegexMap { + return false + } + + u1, _ := url.Parse(s[i].IncomingURL) + u2, _ := url.Parse(s[j].IncomingURL) + if u1 == nil || u2 == nil { + return len(s[i].IncomingURL) < len(s[j].IncomingURL) + } + + if len(u1.Host) == 0 || len(u2.Host) == 0 { + return false + } + + if u1.Host == u2.Host { + if u1.Scheme == u2.Scheme { + if len(u1.Path) == 0 || len(u2.Path) == 0 { + return u1.Path < u2.Path + } + return u1.Path > u2.Path + } + return !(u1.Scheme == "http" || u2.Scheme == "http") + } + + return u1.Path < u2.Path +} diff --git a/tools/voluspa/config_writer.go b/tools/voluspa/config_writer.go new file mode 100644 index 00000000000..73abed44011 --- /dev/null +++ b/tools/voluspa/config_writer.go @@ -0,0 +1,186 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package voluspa + +import ( + "bytes" + "fmt" + "io/ioutil" + "log" + "os" + "path/filepath" + "strings" +) + +type FilesystemWriter struct { + options *Options +} + +func NewFilesystemWriter(options *Options) (*FilesystemWriter, error) { + fw := &FilesystemWriter{options: options} + err := os.MkdirAll(fw.options.Destination, 0755) + if err != nil { + return nil, err + } + return fw, nil +} + +func (f *FilesystemWriter) WriteFiles(files []ManagedFile) error { + groups := make(map[string]interface{}) + for _, mf := range files { + groups[mf.Role] = nil + } + + for _, managedFile := range files { + if _, exists := groups[managedFile.Role]; managedFile.Role != "" && !exists { + continue + } + + filename := fmt.Sprintf("%s/%s", f.options.Destination, managedFile.Filename) + if f.options.Verbose { + log.Printf("Writing %s", filename) + } + + err := os.MkdirAll(filepath.Dir(filename), 0755) + if err != nil { + return err + } + + err = ioutil.WriteFile(filename, managedFile.Contents.Bytes(), 0644) + if err != nil { + return err + } + } + return nil +} + +type ConfigExpander struct { + options *Options +} + +func NewConfigExpander(options *Options) *ConfigExpander { + return &ConfigExpander{options: options} +} + +func (c *ConfigExpander) ExpandRemapConf(parsedConfig *CustomerConfig) ([]ManagedFile, error) { + var managedFiles []ManagedFile + + groups := make(map[string]bool) + for _, config := range parsedConfig.Remaps { + groups[config.Group] = true + } + + configLocations := []ConfigLocation{UnknownLocation} + if parsedConfig.parentChild { + configLocations = []ConfigLocation{ChildConfig, ParentConfig} + } + + for _, configLocation := range configLocations { + for group := range groups { + + var buf bytes.Buffer + buf.WriteString(generatedFileBanner) + + var hasContent bool + for _, config := range parsedConfig.Remaps { + if group != DefaultGroup && config.Group != group { + continue + } + + if configLocation != UnknownLocation && config.ConfigLocation != configLocation { + continue + } + + var roleEnabled bool + if len(config.Role) > 0 { + roleEnabled = true + buf.WriteString(fmt.Sprintf("{%% if salt.pillar.get(\"%s\") %%}\n\n", config.Role)) + } + + buf.WriteString(config.asRemapConf()) + buf.WriteString("\n") + + if roleEnabled { + buf.WriteString("{% endif %}\n\n") + } + + hasContent = true + } + + filename := fmt.Sprintf("%s/%s", strings.ToLower(parsedConfig.property), parsedConfig.remapConfigFilename(group, configLocation)) + + if !hasContent { + return nil, fmt.Errorf("Configuration for %s empty. Not writing %s", parsedConfig.property, filename) + } + + for cdn := range parsedConfig.cdn { + managedFiles = append(managedFiles, NewManagedFile(filename, filename, cdn, parsedConfig.role, parsedConfig.property, &buf, configLocation)) + } + } + } + + return managedFiles, nil +} + +func (c *ConfigExpander) ExpandSubConfigs(parsedConfig *CustomerConfig) ([]ManagedFile, error) { + var managedFiles []ManagedFile + + for _, remap := range parsedConfig.Remaps { + managedSubFiles, err := c.expandSubConfig(parsedConfig, remap) + if err != nil { + return nil, err + } + managedFiles = append(managedFiles, managedSubFiles...) + } + return managedFiles, nil +} + +func (c *ConfigExpander) expandSubConfig(parsedConfig *CustomerConfig, remap Remap) ([]ManagedFile, error) { + var managedFiles []ManagedFile + for i := range remap.mappingRules { + adapter := remap.mappingRules[i] + value := adapter.SubConfigContent + + if len(adapter.SubConfigFileName) == 0 { + continue + } + if value == nil || value.Len() == 0 { + continue + } + + var buf bytes.Buffer + buf.WriteString(generatedFileBanner) + buf.Write(value.Bytes()) + + // append newline if config does not end with one + if value.Bytes()[len(value.Bytes())-1] != '\n' { + buf.WriteByte('\n') + } + + // TODO new method off of PCC? + filename := fmt.Sprintf("%s/%s", strings.ToLower(parsedConfig.property), adapter.SubConfigFileName) + + for cdn := range parsedConfig.cdn { + managedFiles = append(managedFiles, NewManagedFile(filename, filename, cdn, parsedConfig.role, parsedConfig.property, &buf, remap.ConfigLocation)) + } + } + + return managedFiles, nil +} diff --git a/tools/voluspa/consts.go b/tools/voluspa/consts.go new file mode 100644 index 00000000000..d51a0f98c71 --- /dev/null +++ b/tools/voluspa/consts.go @@ -0,0 +1,30 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package voluspa + +const ( + // DefaultTrafficserverConfigurationDir is the default install dir for ATS + DefaultTrafficserverConfigurationDir = "/opt/ats/etc/trafficserver" + + // Child is the label for child hosts, configurations or attributes + Child = "child" + // Parent is the label for parent hosts, configurations, or attributes + Parent = "parent" +) diff --git a/tools/voluspa/customer_config.go b/tools/voluspa/customer_config.go new file mode 100644 index 00000000000..59960d10ae0 --- /dev/null +++ b/tools/voluspa/customer_config.go @@ -0,0 +1,197 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package voluspa + +import ( + "bytes" + "fmt" + "strings" +) + +const ( + // DefaultGroup is the default group name + DefaultGroup = "all" + + DefaultRole = "default" + + remapPrefixBuffer = " " + remapMaxLineLength = 100 + + // generatedFileBanner is the text prepended to all config files + generatedFileBanner = "# Code generated by Voluspa. DO NOT EDIT.\n\n" +) + +// NOTE CustomerConfig and PropertyConfig are closely related. +// Every time I think I want to embed one in the other (or share a common struct), +// I'm reminded that the YAML parser does not handle embedded structs. + +type CustomerConfig struct { + owner string + reference string + lifecycle LifecycleState + property string + role string + filename string + sslCertNames []string + qps int + parentChild bool + parentDomains []string + schemes []string + cdn map[string]interface{} + Remaps []Remap + HAProxyVIPs []HAProxyVIP +} + +type Remap struct { + OriginURL string + IncomingURL string + Group string + Role string + scheme string + RegexMap bool + ConfigLocation ConfigLocation + mappingRules []mappingRule + sourceConfig *ConfigEnvironment + parentOverrides []ParentOverride + parentDomain string +} + +// newCustomerConfig creates a new CustomerConfig from a PropertyConfig +func newCustomerConfig(p *PropertyConfig) (*CustomerConfig, error) { + cc := &CustomerConfig{ + property: p.Property, + sslCertNames: p.SSLCertNames, + parentChild: p.ParentChild, + lifecycle: p.Lifecycle, + qps: p.QPS, + owner: p.Owner, + reference: p.Reference, + role: p.Role, + cdn: make(map[string]interface{}), + schemes: p.Schemes, + HAProxyVIPs: p.HAProxyVIPs, + } + for _, cdn := range p.CDN { + cc.cdn[cdn] = nil + } + return cc, nil +} + +func (c Remap) hasContent() bool { + hasContent := false + for i := range c.mappingRules { + adapter := c.mappingRules[i] + + if adapter.hasContent() { + hasContent = true + } + } + return hasContent +} + +func splitLine(value, prefix string) string { + if len(value) < remapMaxLineLength { + return value + } + + if strings.Contains(value, "{%") { + return value + } + + words := strings.Split(value, " ") + if len(words) < 2 { + return value + } + + out := bytes.Buffer{} + out.WriteString(fmt.Sprintf("%s %s", words[0], words[1])) + + for i := 2; i < len(words); i++ { + if words[i-1] == "@pparam=--config" { + out.WriteString(fmt.Sprintf(" %s", words[i])) + continue + } + out.WriteString(fmt.Sprintf(" \\\n%s %s", prefix, words[i])) + } + return out.String() + +} + +func (c Remap) asRemapConf() string { + var buf bytes.Buffer + if c.RegexMap { + buf.WriteString(fmt.Sprintf("regex_map")) + } else { + buf.WriteString(fmt.Sprintf("map")) + } + buf.WriteString(fmt.Sprintf(" %s \\\n %s", c.IncomingURL, c.OriginURL)) + + if !c.hasContent() { + buf.WriteString("\n") + return buf.String() + } + + buf.WriteString(" \\\n") + + for i := range c.mappingRules { + adapter := c.mappingRules[i] + + if !adapter.hasContent() { + continue + } + + prefixBuffer := remapPrefixBuffer + content := adapter.ConfigContent.String() + if adapter.isCommandLineTemplated() && strings.HasPrefix(content, "{%") { + prefixBuffer = "" + } + + buf.WriteString(fmt.Sprintf("%s%s", prefixBuffer, splitLine(content, prefixBuffer))) + if adapter.isCommandLineTemplated() { + continue + } + + if i < len(c.mappingRules)-1 && c.mappingRules[i+1].hasContent() { + buf.WriteString(" \\") + } + buf.WriteString("\n") + } + + return buf.String() +} + +func (c *CustomerConfig) remapConfigFilename(group string, configLocation ConfigLocation) string { + return configFilename(strings.ToLower(c.property), group, configLocation) +} + +func configFilename(baseFilename, group string, configLocation ConfigLocation) string { + if group == DefaultGroup || group == "" { + if configLocation == ParentConfig { + return fmt.Sprintf("%s_parent.config", baseFilename) + } + return fmt.Sprintf("%s.config", baseFilename) + } + + if configLocation == ParentConfig { + return fmt.Sprintf("%s_%s_parent.config", baseFilename, group) + } + + return fmt.Sprintf("%s_%s.config", baseFilename, group) +} diff --git a/tools/voluspa/doc.go b/tools/voluspa/doc.go new file mode 100644 index 00000000000..59005d41277 --- /dev/null +++ b/tools/voluspa/doc.go @@ -0,0 +1,100 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +/* +Package voluspa generates ATS configuration files for a CDN from a set of +simple YAML files that express the desired behaviour of each of the hosted properties. + +For example, given this property specification: + + schema_version: 1 + owner: Web Group + rcpt: httpbin + + mappings: + - alias: + - httpbin.example.com + origin: https://httpbin.origin.example.com + schemes: [http, https] + + rules: + default: + priority: background + origin_host_header: origin + deny_methods: [ CONNECT ] + +Voluspa will generate the corresponding ATS configuration files in the +specified output directory. A `httpbin/httpbin.config` file: + + # Code generated by Voluspa. DO NOT EDIT. + map http://httpbin.example.com \ + https://https://httpbin.origin.example.com \ + @plugin=conf_remap.so @pparam=proxy.config.url_remap.pristine_host_hdr=0 \ + @plugin=header_rewrite.so @pparam=httpbin/hdrs.config \ + @plugin=propstats.so @pparam=--prefix=proxy.domain.httpbin \ + @action=deny @method=CONNECT + +and a `httpbin/hdrs.config` file: + + # Code generated by Voluspa. DO NOT EDIT. + cond %{REMAP_PSEUDO_HOOK} + set-header @ReceiptService "httpbin{{hosttype}}" + cond %{REMAP_PSEUDO_HOOK} + set-conn-dscp 8 + +The voluspa package enables fine control over the generation of configurations +and enables functionality to be extended via "adapters". + +The distribution includes a `voluspa` command line utility that loads a set of +YAML files and generates the ATS configuration files using code like this: + + import ( + "flag" + "fmt" + "log" + + "github.com/apache/trafficserver/tools/voluspa" + _ "github.com/apache/trafficserver/tools/voluspa/adapters" + ) + + v, err := voluspa.NewVoluspa() + if err != nil { + log.Fatalf("voluspa: %s", err) + } + + // Load the YAML config file for each property in the CDN + for _, filename := range flag.Args() { + err = v.AddConfig(filename) + if err != nil { + log.Fatalf("%s", err) // err includes the filename + } + } + + // Validate the set of configurations, individually and as a whole + if err = v.Validate(true); err != nil { + log.Fatalf("%s", err) + } + + // Generate the resulting ATS config files + if err = v.WriteAllATSFiles(); err != nil { + log.Fatalf("%s", err) + } + +*/ +package voluspa diff --git a/tools/voluspa/haproxy.go b/tools/voluspa/haproxy.go new file mode 100644 index 00000000000..798633e5e8f --- /dev/null +++ b/tools/voluspa/haproxy.go @@ -0,0 +1,81 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package voluspa + +import ( + "bytes" + "fmt" + "strings" +) + +// HAProxyVIP describes a haproxy VIP +type HAProxyVIP struct { + Port int + RIPs []RIP +} + +// RIP is the real IP and Port for a HAProxyVIP +type RIP struct { + IP string + Port int +} + +type HAProxyConfigGenerator struct { +} + +func (h *HAProxyConfigGenerator) Do(parsedConfigs []*CustomerConfig, merged bool) ([]ManagedFile, error) { + var results []ManagedFile + for _, parsedConfig := range parsedConfigs { + if len(parsedConfig.HAProxyVIPs) == 0 { + continue + } + + var buf bytes.Buffer + buf.WriteString(generatedFileBanner) + + for _, hapConfig := range parsedConfig.HAProxyVIPs { + + buf.WriteString(fmt.Sprintf("frontend %s 127.0.0.1:%d\n", parsedConfig.property, hapConfig.Port)) + buf.WriteString(" default_backend app\n") + buf.WriteString(" mode http\n") + buf.WriteString("\n") + buf.WriteString(fmt.Sprintf("backend %s\n", parsedConfig.property)) + buf.WriteString(" mode http\n") + buf.WriteString(" balance roundrobin\n") + buf.WriteString(" option httpchk GET / HTTP/1.1\\r\\nHost:localhost\n") + + for i, rip := range hapConfig.RIPs { + buf.WriteString(fmt.Sprintf(" server %s%03d %s:%d check\n", parsedConfig.property, i+1, rip.IP, rip.Port)) + } + + filename := fmt.Sprintf("%s/%s.hap.config", strings.ToLower(parsedConfig.property), strings.ToLower(parsedConfig.property)) + + results = append(results, ManagedFile{ + Filename: filename, + Role: "", + Property: parsedConfig.property, + Contents: &buf, + ConfigType: UnknownLocation, + }) + } + } + + return results, nil +} diff --git a/tools/voluspa/hosting_config.go b/tools/voluspa/hosting_config.go new file mode 100644 index 00000000000..315dfe00c1e --- /dev/null +++ b/tools/voluspa/hosting_config.go @@ -0,0 +1,213 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package voluspa + +import ( + "bytes" + "fmt" + "net/url" + "sort" + "strings" +) + +type HostingConfigurator struct { + options *Options +} + +const ( + ramVolString string = "{{ ramdisk_volume }}" + defaultVolString string = "{{ default_volumes }}" + + HostingConfigDefaultFilename = "hosting.config_default" +) + +type StorageType int + +const ( + ramVolume StorageType = iota + diskVolume +) + +// hostingHostnameMap maps hostnames to StorageType +type hostingHostnameMap map[string]StorageType + +type hostingHostname struct { +} + +func newHostingHostname(hn string) *hostingHostname { + return &hostingHostname{} +} + +func newHostingConfigurator(options *Options) *HostingConfigurator { + return &HostingConfigurator{ + options: options, + } +} + +func (h *HostingConfigurator) Do(parsedConfigs []*CustomerConfig, merge bool) ([]ManagedFile, error) { + if merge { + return h.get(parsedConfigs, NoCDN) + } + + // otherwise, group properties by CDN and generate hosting.config for each CDN separately + grouped := groupCustomerConfigsByCDN(parsedConfigs) + + var managedFiles []ManagedFile + for cdn, configs := range grouped { + files, err := h.get(configs, cdn) + if err != nil { + return nil, err + } + managedFiles = append(managedFiles, files...) + } + + return managedFiles, nil + +} + +func (h *HostingConfigurator) get(parsedConfigs []*CustomerConfig, cdn string) ([]ManagedFile, error) { + + hostingHostnames := make(map[string]hostingHostnameMap) + for _, parsedConfig := range parsedConfigs { + role := parsedConfig.role + if hostingHostnames[role] == nil { + hostingHostnames[role] = make(hostingHostnameMap) + } + for _, remap := range parsedConfig.Remaps { + destURL := remap.OriginURL + if remap.sourceConfig.RemapOptions.HasOptionSet("origin_host_header") { + value, err := remap.sourceConfig.RemapOptions.ValueByNameAsString("origin_host_header") + if err != nil { + return nil, err + } + if value == "alias" { + destURL = remap.IncomingURL + } + } + u, err := url.Parse(destURL) + if err != nil { + return nil, err + } + storageConfig := "disk_volume" // default + if remap.sourceConfig.RemapOptions.HasOptionSet("storage_volume") { + value, err := remap.sourceConfig.RemapOptions.ValueByNameAsString("storage_volume") + if err != nil { + return nil, err + } + storageConfig = value + } + if storageConfig == "ramdisk_volume" { + hostingHostnames[role][u.Host] = ramVolume + } else { + hostingHostnames[role][u.Host] = diskVolume + } + } + } + + return h.ExpandConfigTemplate(hostingHostnames, cdn) +} + +func (h *HostingConfigurator) ExpandConfigTemplate(hostingHostnamesByRole map[string]hostingHostnameMap, cdn string) ([]ManagedFile, error) { + seen := make(map[string]bool) + + var buf bytes.Buffer + buf.WriteString(generatedFileBanner) + buf.WriteString(fmt.Sprintf("{%% if salt.pillar.get(\"%s\") %%}\n", h.options.RamDiskRole)) + + var roles []string + for k := range hostingHostnamesByRole { + roles = append(roles, k) + } + sort.Strings(roles) + + for _, role := range roles { + hostingHostnames := hostingHostnamesByRole[role] + + var hostnames []string + for hostname := range hostingHostnames { + hostnames = append(hostnames, hostname) + } + + if len(hostnames) == 0 { + continue + } + + sort.Strings(hostnames) + + hasRamVolumeHosts := false + for _, hostname := range hostnames { + storage := hostingHostnames[hostname] + if storage == ramVolume { + hasRamVolumeHosts = true + break + } + } + + if !hasRamVolumeHosts { + continue + } + + buf.WriteString(h.startRoleGuard(role)) + + for _, hostname := range hostnames { + if _, exists := seen[hostname]; exists { + continue + } + + storage := hostingHostnames[hostname] + if storage == ramVolume { + seen[hostname] = true + buf.WriteString(fmt.Sprintf("hostname=%s %s\n", hostname, ramVolString)) + } + } + buf.WriteString(h.endRoleGuard(role)) + } + + buf.WriteString(fmt.Sprintf("hostname=* %s\n", defaultVolString)) + + buf.WriteString("{% else %}\n# hosting.config disabled on this host\n{% endif %}\n") + + filename := HostingConfigDefaultFilename + if len(cdn) > 0 && (cdn != DefaultCDN && cdn != NoCDN) { + filename = fmt.Sprintf("hosting.config_%s", cdn) + } + + return []ManagedFile{NewManagedFile(filename, "hosting.config", cdn, "", "", &buf, UnknownLocation)}, nil + +} + +func (h *HostingConfigurator) startRoleGuard(role string) string { + if len(role) == 0 { + return "" + } + + roleName := role + if !strings.HasPrefix(role, "roles_") { + roleName = fmt.Sprintf("roles_%s", role) + } + return fmt.Sprintf("{%% if salt.pillar.get('%s') %%}\n", roleName) +} + +func (h *HostingConfigurator) endRoleGuard(role string) string { + if len(role) == 0 { + return "" + } + return "{% endif %}\n\n" +} diff --git a/tools/voluspa/internal/util/regex/regex.go b/tools/voluspa/internal/util/regex/regex.go new file mode 100644 index 00000000000..f5717b40687 --- /dev/null +++ b/tools/voluspa/internal/util/regex/regex.go @@ -0,0 +1,24 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package regex + +type Validator interface { + IsValid(in string) (bool, error) +} diff --git a/tools/voluspa/internal/util/regex/regex_test.go b/tools/voluspa/internal/util/regex/regex_test.go new file mode 100644 index 00000000000..ad438a75b92 --- /dev/null +++ b/tools/voluspa/internal/util/regex/regex_test.go @@ -0,0 +1,45 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package regex + +import ( + "testing" +) + +func TestGoLangRegex(t *testing.T) { + testCases := []struct { + u string + e bool + }{ + {u: "asdf", e: true}, + {u: `^[-_A-Za-z0-9\.]+$`, e: true}, + {u: "", e: true}, + {u: " ^/([^/]*-eu-dub-[^/]*)/(.*) https://s3-eu-west-1.amazonaws.com$0", e: true}, + {u: "[", e: false}, + } + + glr := &GoLangRegex{} + for _, tc := range testCases { + e, err := glr.IsValid(tc.u) + if e != tc.e { + t.Errorf("%q: expected %t got %t: %s", tc.u, tc.e, e, err) + } + } +} diff --git a/tools/voluspa/internal/util/regex/vregex.go b/tools/voluspa/internal/util/regex/vregex.go new file mode 100644 index 00000000000..cde553967f2 --- /dev/null +++ b/tools/voluspa/internal/util/regex/vregex.go @@ -0,0 +1,33 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package regex + +import ( + "regexp" +) + +// GoLangRegex implements regex.Validator interface using go's builtin regex methods +type GoLangRegex struct { +} + +func (e *GoLangRegex) IsValid(in string) (bool, error) { + _, err := regexp.Compile(in) + return err == nil, err +} diff --git a/tools/voluspa/lifecycle.go b/tools/voluspa/lifecycle.go new file mode 100644 index 00000000000..c59186feaa3 --- /dev/null +++ b/tools/voluspa/lifecycle.go @@ -0,0 +1,29 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package voluspa + +type LifecycleState string + +const ( + UnknownLifecycleState LifecycleState = "" + Onboarding LifecycleState = "onboarding" + Live LifecycleState = "live" + Retired LifecycleState = "retired" +) diff --git a/tools/voluspa/managed_file.go b/tools/voluspa/managed_file.go new file mode 100644 index 00000000000..48971622998 --- /dev/null +++ b/tools/voluspa/managed_file.go @@ -0,0 +1,51 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package voluspa + +import ( + "bytes" + "fmt" +) + +type ManagedFile struct { + Filename string + RemoteFilename string + Role string + CDN string + Property string + Contents *bytes.Buffer + ConfigType ConfigLocation +} + +func NewManagedFile(filename, remoteFilename, cdn, role, property string, contents *bytes.Buffer, configType ConfigLocation) ManagedFile { + return ManagedFile{ + Filename: filename, + RemoteFilename: remoteFilename, + Role: role, + CDN: cdn, + Property: property, + Contents: contents, + ConfigType: configType, + } +} + +func (mf ManagedFile) String() string { + return fmt.Sprintf("Filename=%s RFN=%s Role=%s CDN=%s Property=%s Size=%d Type=%d", mf.Filename, mf.RemoteFilename, mf.Role, mf.CDN, mf.Property, mf.Contents.Len(), mf.ConfigType) +} diff --git a/tools/voluspa/options.go b/tools/voluspa/options.go new file mode 100644 index 00000000000..6888ede3903 --- /dev/null +++ b/tools/voluspa/options.go @@ -0,0 +1,30 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package voluspa + +type Options struct { + Verbose bool + SkipSchemaValidation bool + PromoteRolesToCDN bool + Destination string + DefaultsLocation string + SchemaLocation string + RamDiskRole string +} diff --git a/tools/voluspa/parent_config.go b/tools/voluspa/parent_config.go new file mode 100644 index 00000000000..5b825d12080 --- /dev/null +++ b/tools/voluspa/parent_config.go @@ -0,0 +1,393 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package voluspa + +import ( + "bytes" + "fmt" + "net/url" + "sort" + "strings" +) + +// parentDomainMap maps domain names to parentDomain struct +type parentDomainMap map[string]*parentDomain + +type ParentConfigurator struct { + options *Options +} + +type Override struct { + schemes map[string]bool + primaryList []string + secondaryList []string + HTTPPort int + HTTPSPort int + strategy string + goDirect bool + ignoreQueryString bool + overrideDomain string +} + +type parentDomain struct { + originalDomain string + schemes map[string]bool + ignoreQueryString bool + OverridesByRole map[string]Override +} + +func newParentDomain(domain string) *parentDomain { + return &parentDomain{originalDomain: domain, schemes: make(map[string]bool), OverridesByRole: make(map[string]Override)} +} + +func newParentConfigurator(options *Options) *ParentConfigurator { + return &ParentConfigurator{ + options: options, + } +} + +func (p *ParentConfigurator) Do(parsedConfigs []*CustomerConfig, merge bool) ([]ManagedFile, error) { + if merge { + return p.get(parsedConfigs, NoCDN) + } + + // otherwise, group properties by CDN and generate parent.config for each CDN separately + grouped := groupCustomerConfigsByCDN(parsedConfigs) + + var managedFiles []ManagedFile + for cdn, configs := range grouped { + files, err := p.get(configs, cdn) + if err != nil { + return nil, err + } + managedFiles = append(managedFiles, files...) + } + + return managedFiles, nil + +} + +func (p *ParentConfigurator) get(parsedConfigs []*CustomerConfig, cdn string) ([]ManagedFile, error) { + // a map of roles to parentDomainMaps + parentDomains := make(map[string]parentDomainMap) + for _, parsedConfig := range parsedConfigs { + + role := parsedConfig.role + if _, exists := parentDomains[role]; !exists { + parentDomains[role] = make(parentDomainMap) + } + + for _, domain := range parsedConfig.parentDomains { + if parsedConfig.lifecycle == Retired { + continue + } + + for _, remap := range parsedConfig.Remaps { + if remap.ConfigLocation == ParentConfig { + continue + } + + pd, ok := parentDomains[role][domain] + if !ok { + pd = newParentDomain(domain) + parentDomains[role][domain] = pd + } + + // extract schemes from toplevel or from remap rules + if len(remap.scheme) > 0 { + pd.schemes[remap.scheme] = true + } else { + for _, scheme := range parsedConfig.schemes { + pd.schemes[scheme] = true + } + } + + scheme, err := extractScheme(remap.IncomingURL) + if err != nil { + return nil, fmt.Errorf("could not extract scheme from %q: %s", remap.IncomingURL, err) + } + + if len(scheme) > 0 { + pd.schemes[scheme] = true + } + + if len(remap.parentOverrides) > 0 && remap.parentDomain == pd.originalDomain { + for _, o := range remap.parentOverrides { + ignore := o.IgnoreQuerystring + if hasRemoveAllParamsOption(remap.sourceConfig.RemapOptions) { + ignore = true + } + pd.OverridesByRole[o.Role] = Override{ + primaryList: o.PrimaryList, + secondaryList: o.SecondaryList, + HTTPPort: o.HTTPPort, + HTTPSPort: o.HTTPSPort, + ignoreQueryString: ignore, + strategy: o.Strategy, + goDirect: o.GoDirect, + overrideDomain: o.DestDomain, + } + } + } + + if remap.sourceConfig.RemapOptions == nil { + continue + } + + // We obey if it was set for any remap + if hasRemoveAllParamsOption(remap.sourceConfig.RemapOptions) || hasIgnoreQuery(remap.sourceConfig.RemapOptions) { + pd.ignoreQueryString = true + } + + } + } + } + + return p.ExpandConfigTemplate(parentDomains, cdn) +} + +func extractScheme(in string) (string, error) { + parsedURL, err := url.Parse(in) + if err == nil { + return parsedURL.Scheme, nil + } + + if strings.HasPrefix(in, "https:") { + return "https", nil + } + + if strings.HasPrefix(in, "http:") { + return "http", nil + } + + return "", nil +} + +func hasRemoveAllParamsOption(remapOptions RemapOptions) bool { + vals, err := remapOptions.ValueByNameAsStringMapInterface("cachekey") + if err != nil { + return false + } + + val, ok := vals["remove_all_params"] + if !ok { + return false + } + + optionValue, ok := val.(bool) + if !ok { + return false + } + + return optionValue +} + +func hasIgnoreQuery(remapOptions RemapOptions) bool { + vals, err := remapOptions.ValueByNameAsStringMapInterface("cachekey") + if err != nil { + return false + } + + val, ok := vals["parent_selection"] + if !ok { + return false + } + + optionValue, ok := val.(string) + if !ok { + return false + } + return optionValue == "ignore_query" +} + +func (p *ParentConfigurator) ExpandConfigTemplate(parentDomains map[string]parentDomainMap, cdn string) ([]ManagedFile, error) { + var files []ManagedFile + for role := range parentDomains { + parentDomains, ok := parentDomains[role] + if !ok { + return nil, fmt.Errorf("role %q not found", role) + } + + mf, err := p.expandConfigTemplate(parentDomains, cdn, role) + if err != nil { + return nil, err + } + + if mf == nil { + continue + } + + files = append(files, *mf) + + // ATS wants a file on parent hosts. match name that's used in salt + filename := "parent_empty.config" + if len(role) > 0 { + filename = fmt.Sprintf("parent_empty.config_%s", role) + } + + emptyParentConfig := NewManagedFile(filename, "parent.config", "", role, "", intentionallyEmptyMessage, ParentConfig) + files = append(files, emptyParentConfig) + } + + return files, nil +} + +// DomainNames allows for implementing the sort interface; regular string sort is not good enough +// we want the longest match to win in the case of bar.com and foo.bar.com. +type DomainNames []string + +// var _ sort.Interface = DomainNames{} + +func (d DomainNames) Len() int { + return len(d) +} +func (d DomainNames) Swap(i, j int) { + d[i], d[j] = d[j], d[i] +} + +func (d DomainNames) Less(i, j int) bool { + + if len(strings.Split(d[i], ".")) != len(strings.Split(d[j], ".")) { + return len(strings.Split(d[i], ".")) > len(strings.Split(d[j], ".")) + } + if strings.Compare(d[i], d[j]) < 0 { + return true + } + return false +} + +var intentionallyEmptyMessage = bytes.NewBufferString("# This file is intentionally left blank\n") + +func (p *ParentConfigurator) expandConfigTemplate(parentDomains parentDomainMap, cdn, role string) (*ManagedFile, error) { + seen := make(map[string]bool) + + var buf bytes.Buffer + buf.WriteString("\n") + + buf.WriteString(generatedFileBanner) + var domains DomainNames + for _, pd := range parentDomains { + domains = append(domains, pd.originalDomain) + } + sort.Sort(domains) + + for _, domain := range domains { + + if _, exists := seen[domain]; exists { + continue + } + + pd := parentDomains[domain] + + var schemes []string + for scheme := range pd.schemes { + schemes = append(schemes, scheme) + } + sort.Strings(schemes) + + for _, scheme := range schemes { + if len(pd.OverridesByRole) > 0 { + + // sort roles to make diffs more predictable + roles := make([]string, 0) + for role := range pd.OverridesByRole { + roles = append(roles, role) + } + sort.Strings(roles) + + for _, role := range roles { + override := pd.OverridesByRole[role] + port := override.HTTPPort + if scheme == "https" { + port = override.HTTPSPort + } + pstr := fmt.Sprintf(`parent="{{ %s_parents }}" {{ secondary_%s_parents }}`, scheme, scheme) // the default + if len(override.primaryList) > 0 { + primaryString := "parent=\"" + secondaryString := "secondary_parent=\"" + for _, h := range override.primaryList { + primaryString += fmt.Sprintf("%s:%d,", h, port) + } + primaryString = strings.TrimSuffix(primaryString, ",") + primaryString += "\"" + if len(override.secondaryList) > 0 { + for _, h := range override.secondaryList { + secondaryString += fmt.Sprintf("%s:%d,", h, port) + } + secondaryString = strings.TrimSuffix(secondaryString, ",") + secondaryString += "\"" + } else { + secondaryString = "" + } + pstr = fmt.Sprintf("%s %s", primaryString, secondaryString) + } + if role != "" { + buf.WriteString(fmt.Sprintf("\n{%% if salt.pillar.get(\"%s\") %%}\n", role)) + } + strategy := override.strategy + if strategy == "" { + strategy = "consistent_hash" + } + direct := override.goDirect // false is default? + finalDomain := domain + if override.overrideDomain != "" { + finalDomain = override.overrideDomain + } + buf.WriteString(fmt.Sprintf(`dest_domain=%s scheme=%s %s round_robin=%s go_direct=%t`, finalDomain, scheme, pstr, strategy, direct)) + if override.ignoreQueryString { + buf.WriteString(" qstring=ignore") + } + buf.WriteString("\n") + if role != "" { + buf.WriteString("{% endif %}\n\n") + } + } + } else { + // This is the "legacy way" with parent_dest_domwain at the mapping level NOTE DEPRECATED + buf.WriteString(fmt.Sprintf( + `dest_domain=%s scheme=%s parent="{{ %s_parents }}" {{ secondary_%s_parents }} round_robin=consistent_hash go_direct=false`, domain, scheme, scheme, scheme)) + if pd.ignoreQueryString { + buf.WriteString(" qstring=ignore") + } + buf.WriteString("\n") + } + } + seen[domain] = true + } + + buf.WriteString("\n") + + if len(seen) == 0 { + return nil, nil + } + + filename := "parent.config_default" + if len(role) > 0 { + filename = fmt.Sprintf("parent.config_%s", role) + } + + if cdn != DefaultCDN && cdn != NoCDN { + filename = fmt.Sprintf("parent.config_%s", cdn) + } + + mf := NewManagedFile(filename, "parent.config", cdn, role, "", &buf, ChildConfig) + + return &mf, nil +} diff --git a/tools/voluspa/parent_config_test.go b/tools/voluspa/parent_config_test.go new file mode 100644 index 00000000000..b4558690cfd --- /dev/null +++ b/tools/voluspa/parent_config_test.go @@ -0,0 +1,47 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package voluspa + +import ( + "testing" +) + +func TestExtractScheme(t *testing.T) { + testCases := []struct { + in string + out string + }{ + {"https://store-(fred|123|test).domain.com/", "https"}, + {"http://store-(bob|test).domain.com/", "http"}, + {"http://www.domain.com/", "http"}, + {"www.domain.com/", ""}, + } + for i, tc := range testCases { + scheme, err := extractScheme(tc.in) + if err != nil { + t.Errorf("test #%d: unexpected error: %s", i, err) + continue + } + + if scheme != tc.out { + t.Errorf("test #%d: Sorted order expected %q got %q", i, tc.out, scheme) + } + } +} diff --git a/tools/voluspa/property_config.go b/tools/voluspa/property_config.go new file mode 100644 index 00000000000..fb327b6382f --- /dev/null +++ b/tools/voluspa/property_config.go @@ -0,0 +1,326 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package voluspa + +import ( + "fmt" + "io/ioutil" + "net/url" + "os" + "regexp" + "sort" + "unicode" + + yaml "gopkg.in/yaml.v2" +) + +// PropertyConfig represents the raw parsed structure from YAML +type PropertyConfig struct { + Owner string `json:"owner,omitempty" yaml:",omitempty"` + Reference string `json:"reference,omitempty" yaml:",omitempty"` + Lifecycle LifecycleState `json:"lifecycle,omitempty" yaml:",omitempty"` + Role string `json:"role,omitempty" yaml:",omitempty"` + Property string `json:"rcpt,omitempty" yaml:"rcpt,omitempty"` + SchemaVersion string `json:"schema_version,omitempty" yaml:"schema_version,omitempty"` + SSLCertNames []string `json:"ssl_cert_names,omitempty" yaml:"ssl_cert_names,omitempty"` + Schemes []string `json:"schemes,omitempty" yaml:",omitempty"` + CDN []string `json:"cdn,omitempty" yaml:",omitempty"` + Org string `json:"org,omitempty" yaml:"org,omitempty"` + Rules map[string]RemapOptions + ExplicitMappings []*Mapping `json:"explicit,omitempty" yaml:"explicit,omitempty"` + Mappings []*Mapping `json:"mappings,omitempty" yaml:"mappings,omitempty"` + ParentChild bool `json:"parent_child,omitempty" yaml:"parent_child,omitempty"` + QPS int `json:"qps" yaml:"qps"` + HAProxyVIPs []HAProxyVIP `json:"ha_proxy,omitempty" yaml:"ha_proxy,omitempty"` + Tests map[string]CDNTest `json:"tests,omitempty" yaml:"tests,omitempty"` + Receiptsd *ReceiptsdConfig `json:"receiptsd,omitempty" yaml:"receiptsd,omitempty"` +} + +type Mapping struct { + Origin string `json:"origin,omitempty" yaml:"origin,omitempty"` + Group string `json:"group,omitempty" yaml:"group,omitempty"` + Role string `json:"role,omitempty" yaml:"role,omitempty"` + Alias []string `json:"alias,omitempty" yaml:"alias,omitempty"` + Schemes []string `json:"schemes,omitempty" yaml:"schemes,omitempty"` + + RuleName string `json:"rule" yaml:"rule"` + ParentDestDomain string `json:"parent_dest_domain,omitempty" yaml:"parent_dest_domain,omitempty"` + ParentOverride []ParentOverride `json:"parent_override,omitempty" yaml:"parent_override,omitempty"` + RegexMap bool `json:"regex_map" yaml:"regex_map"` + RemapOptions RemapOptions `json:",omitempty" yaml:",omitempty"` + explicit bool + id int +} + +type ParentOverride struct { + Role string `json:"role,omitempty" yaml:"role,omitempty"` + DestDomain string `json:"dest_domain,omitempty" yaml:"dest_domain"` + PrimaryList []string `json:"primary_list,omitempty" yaml:"primary_list,omitempty"` + SecondaryList []string `json:"secondary_list,omitempty" yaml:"secondary_list,omitempty"` + Strategy string `json:"strategy,omitempty" yaml:"strategy,omitempty"` + IgnoreQuerystring bool `json:"ignore_querystring,omitempty" yaml:"ignore_querystring,omitempty"` + HTTPPort int `json:"http_port,omitempty" yaml:"http_port,omitempty"` + HTTPSPort int `json:"https_port,omitempty" yaml:"https_port,omitempty"` + GoDirect bool `json:"go_direct,omitempty" yaml:"go_direct,omitempty"` +} + +func (m *Mapping) ParentDomain() string { + + // a po without a role but with a domain + for _, po := range m.ParentOverride { + if po.Role == "" && len(po.DestDomain) > 0 { + return po.DestDomain + } + } + + // Deprecated + if len(m.ParentDestDomain) > 0 { + return m.ParentDestDomain + } + // the domain is not overriden, old style or new style, return the origin domain + parentURL, err := url.Parse(m.Origin) + if err == nil && len(parentURL.Scheme) > 0 { + return parentURL.Host + } + + return m.Origin +} + +const ( + NoCDN = "" + NoRole = "" + DefaultCDN = "edge" + DefaultScheme = "http" + DefaultRulesetName = "default" +) + +var propertyNameRE = regexp.MustCompile(`^[-_A-Za-z0-9\.]+$`) + +func isValidPropertyName(in string) bool { + return propertyNameRE.MatchString(in) +} + +// Setup validates and sets up a PropertyConfig +func (p *PropertyConfig) Setup() error { + + if len(p.Schemes) > 0 { + for _, scheme := range p.Schemes { + if !isValidScheme(scheme) { + return fmt.Errorf("invalid/unsupported scheme %q", scheme) + } + } + } + + // rcpt is the public-facing name of the field, Property is the internal name + if len(p.Property) == 0 { + return fmt.Errorf("rcpt is required") + } + + if !isValidPropertyName(p.Property) { + return fmt.Errorf("property name can only contain alphanumeric characters") + } + + // assign an id for each rule name + // sort keys/path for stable ids + matchIds := make(map[string]int) + { + var keys []string + for k := range p.Rules { + keys = append(keys, k) + } + sort.Strings(keys) + + i := 1 + for _, k := range keys { + matchIds[k] = i + i++ + } + } + + for _, m := range p.Mappings { + if m.RuleName == "" { + if _, ok := p.Rules["default"]; ok { + m.RuleName = "default" + } + } + + if m.Group == "" { + m.Group = DefaultGroup + } + + m.id = matchIds[m.RuleName] + + if ruleset, ok := p.Rules[m.RuleName]; ok { + m.RemapOptions = ruleset + } else { + return fmt.Errorf("invalid ruleset name: %q for origin %q", m.RuleName, m.Origin) + } + + if len(m.Alias) == 0 { + return fmt.Errorf("no alias specified for origin %q", m.Origin) + } + if hasLoop(m) { + return fmt.Errorf("alias and origin %q will loop", m.Origin) + } + } + + for _, m := range p.ExplicitMappings { + if m.RuleName == "" { + if _, ok := p.Rules[DefaultRulesetName]; ok { + m.RuleName = DefaultRulesetName + } + } + + if m.Group == "" { + m.Group = DefaultGroup + } + + m.id = matchIds[m.RuleName] + + if ruleset, ok := p.Rules[m.RuleName]; ok { + m.RemapOptions = ruleset + } else { + return fmt.Errorf("invalid ruleset name: %q for origin %q", m.RuleName, m.Origin) + } + + if len(m.Alias) == 0 { + return fmt.Errorf("no alias specified for origin %q", m.Origin) + } + } + + return nil +} + +func hasLoop(m *Mapping) bool { + for _, scheme := range m.schemes() { + for _, incoming := range m.Alias { + incomingURL := formatURL(scheme, incoming) + originURL := formatURL(scheme, m.Origin) + if incomingURL == originURL { + return true + } + } + } + return false +} + +func (p *PropertyConfig) loadDefaultConfig(defaultsLocation, filename string) error { + var fullPath string + if len(defaultsLocation) > 0 { + fullPath = fmt.Sprintf("%s/%s", defaultsLocation, filename) + } else { + fullPath = filename + } + if _, err := os.Stat(fullPath); os.IsNotExist(err) { + return nil + } + + cfgContents, err := ioutil.ReadFile(fullPath) + if err != nil { + return err + } + + return p.parseConfig(cfgContents) +} + +func isASCII(in string) (bool, rune) { + for _, c := range in { + if c > unicode.MaxASCII { + return false, c + } + } + return true, 0 +} + +func (p *PropertyConfig) parseConfig(cfgContents []byte) error { + valid, char := isASCII(string(cfgContents)) + if !valid { + return fmt.Errorf("non-ASCII characters not supported: invalid rune %q", string(char)) + } + + return yaml.Unmarshal(cfgContents, &p) +} + +func (p *PropertyConfig) mergeConfig(c *PropertyConfig) { + overrides, ok := c.Rules[DefaultRulesetName] + if !ok { + return + } + + for _, ruleset := range p.Rules { + if ruleset == nil { + continue + } + for k, v := range overrides { + if _, exists := ruleset[k]; !exists { + ruleset[k] = v + } + } + } +} + +func newPropertyConfig() *PropertyConfig { + return &PropertyConfig{ + Rules: make(map[string]RemapOptions), + Schemes: []string{DefaultScheme}, + CDN: []string{DefaultCDN}, + Lifecycle: Live, + } +} + +// NewPropertyConfig creates a new PropertyConfig from a buffer containing a YAML file +func NewPropertyConfig(buffer []byte) (*PropertyConfig, error) { + return NewPropertyConfigWithDefaults(buffer, "") +} + +// NewPropertyConfigWithDefaults creates a new PropertyConfig from a buffer containing a YAML file +func NewPropertyConfigWithDefaults(buffer []byte, defaultsLocation string) (*PropertyConfig, error) { + cfg := newPropertyConfig() + + if err := cfg.parseConfig(buffer); err != nil { + return nil, fmt.Errorf("Could not load YAML file. err=%s", err) + } + + return inflatePropertyConfig(cfg, defaultsLocation) +} + +func inflatePropertyConfig(cfg *PropertyConfig, defaultsLocation string) (*PropertyConfig, error) { + dcfg := newPropertyConfig() + + if err := dcfg.loadDefaultConfig(defaultsLocation, "default.conf"); err != nil { + return nil, fmt.Errorf("Could not load %q: %s", "defaults/default.conf", err) + } + + cfg.mergeConfig(dcfg) + + if cfg.ParentChild { + dpCfg := newPropertyConfig() + + if err := dpCfg.loadDefaultConfig(defaultsLocation, "default_parent.conf"); err != nil { + return nil, fmt.Errorf("Could not load %q: %s", "defaults/default_parent.conf", err) + } + cfg.mergeConfig(dpCfg) + } + + if err := cfg.Setup(); err != nil { + return nil, err + } + return cfg, nil +} diff --git a/tools/voluspa/property_remaps.go b/tools/voluspa/property_remaps.go new file mode 100644 index 00000000000..19685ab70a7 --- /dev/null +++ b/tools/voluspa/property_remaps.go @@ -0,0 +1,168 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package voluspa + +import ( + "bytes" + "fmt" + "strings" +) + +type PropertyRemapGenerator struct { + options *Options +} + +func newPropertyRemapGenerator(options *Options) *PropertyRemapGenerator { + return &PropertyRemapGenerator{options: options} +} + +func (p *PropertyRemapGenerator) Do(parsedConfigs []*CustomerConfig, merge bool) ([]ManagedFile, error) { + if len(parsedConfigs) == 0 { + return nil, ErrMinimumConfigsNotMet + } + + var managedFiles []ManagedFile + for _, parsedConfig := range parsedConfigs { + if parsedConfig.lifecycle == Retired { + continue + } + expanded, err := p.expandRemapConf(parsedConfig) + if err != nil { + return nil, fmt.Errorf("Could not write remap config for property %q: %s", parsedConfig.property, err) + } + + managedFiles = append(managedFiles, expanded...) + + expanded, err = p.expandSubConfigs(parsedConfig) + if err != nil { + return nil, fmt.Errorf("Could not write subconfigs for property %q: %s", parsedConfig.property, err) + } + + managedFiles = append(managedFiles, expanded...) + } + + return managedFiles, nil +} + +func (p *PropertyRemapGenerator) expandRemapConf(parsedConfig *CustomerConfig) ([]ManagedFile, error) { + var managedFiles []ManagedFile + + groups := make(map[string]bool) + for _, config := range parsedConfig.Remaps { + groups[config.Group] = true + } + + configLocations := []ConfigLocation{UnknownLocation} + if parsedConfig.parentChild { + configLocations = []ConfigLocation{ChildConfig, ParentConfig} + } + + for _, configLocation := range configLocations { + for group := range groups { + + var buf bytes.Buffer + buf.WriteString(generatedFileBanner) + + var hasContent bool + for _, config := range parsedConfig.Remaps { + if group != DefaultGroup && config.Group != group { + continue + } + + if configLocation != UnknownLocation && config.ConfigLocation != configLocation { + continue + } + + var roleEnabled bool + if len(config.Role) > 0 { + roleEnabled = true + buf.WriteString(fmt.Sprintf("{%% if salt.pillar.get(\"%s\") %%}\n\n", config.Role)) + } + + buf.WriteString(config.asRemapConf()) + buf.WriteString("\n") + + if roleEnabled { + buf.WriteString("{% endif %}\n\n") + } + + hasContent = true + } + + filename := fmt.Sprintf("%s/%s", strings.ToLower(parsedConfig.property), parsedConfig.remapConfigFilename(group, configLocation)) + + if !hasContent { + return nil, fmt.Errorf("Configuration for %s empty. Not writing %s", parsedConfig.property, filename) + } + + for cdn := range parsedConfig.cdn { + managedFiles = append(managedFiles, NewManagedFile(filename, filename, cdn, parsedConfig.role, parsedConfig.property, &buf, configLocation)) + } + } + } + + return managedFiles, nil +} + +func (p *PropertyRemapGenerator) expandSubConfigs(parsedConfig *CustomerConfig) ([]ManagedFile, error) { + var managedFiles []ManagedFile + + for _, remap := range parsedConfig.Remaps { + managedSubFiles, err := p.expandSubConfig(parsedConfig, remap) + if err != nil { + return nil, err + } + managedFiles = append(managedFiles, managedSubFiles...) + } + return managedFiles, nil +} + +func (p *PropertyRemapGenerator) expandSubConfig(parsedConfig *CustomerConfig, remap Remap) ([]ManagedFile, error) { + var managedFiles []ManagedFile + for i := range remap.mappingRules { + adapter := remap.mappingRules[i] + value := adapter.SubConfigContent + + if len(adapter.SubConfigFileName) == 0 { + continue + } + if value == nil || value.Len() == 0 { + continue + } + + var buf bytes.Buffer + buf.WriteString(generatedFileBanner) + buf.Write(value.Bytes()) + + // append newline if config does not end with one + if value.Bytes()[len(value.Bytes())-1] != '\n' { + buf.WriteByte('\n') + } + + // TODO new method off of PCC? + filename := fmt.Sprintf("%s/%s", strings.ToLower(parsedConfig.property), adapter.SubConfigFileName) + + for cdn := range parsedConfig.cdn { + managedFiles = append(managedFiles, NewManagedFile(filename, filename, cdn, parsedConfig.role, parsedConfig.property, &buf, remap.ConfigLocation)) + } + } + + return managedFiles, nil +} diff --git a/tools/voluspa/receiptsd.go b/tools/voluspa/receiptsd.go new file mode 100644 index 00000000000..6ee8434bae8 --- /dev/null +++ b/tools/voluspa/receiptsd.go @@ -0,0 +1,32 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package voluspa + +// ReceiptsdConfig is the config for receiptsd managed by voluspa +type ReceiptsdConfig struct { + IgnoredExtensions []string `json:"ignored_extensions,omitempty" yaml:"ignored_extensions"` + + Sampling *struct { + Posters Posters `json:"posters,omitempty" yaml:"posters"` + } `json:"sampling,omitempty"` +} + +// Posters is a key/value map of poster name to % to post +type Posters map[string]string diff --git a/tools/voluspa/registry.go b/tools/voluspa/registry.go new file mode 100644 index 00000000000..9bbcbf5a54e --- /dev/null +++ b/tools/voluspa/registry.go @@ -0,0 +1,132 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package voluspa + +import ( + "fmt" + "log" +) + +var AdaptersRegistry *AdapterRegistry + +func init() { + AdaptersRegistry = NewAdapterRegistry() +} + +// AdapterRegistry is the central registry for all plugin config plugins +type AdapterRegistry struct { + adapters map[AdapterType]Adapter + compoundAdapters map[AdapterType][]Adapter + configNameAdapterMap map[string]AdapterType + fileNameMap map[string]AdapterType +} + +func NewAdapterRegistry() *AdapterRegistry { + return &AdapterRegistry{ + adapters: make(map[AdapterType]Adapter), + compoundAdapters: make(map[AdapterType][]Adapter), + configNameAdapterMap: make(map[string]AdapterType), + fileNameMap: make(map[string]AdapterType), + } +} + +func (r *AdapterRegistry) AddAdapter(p Adapter) { + if p.PluginType() == CompoundAdapter { + r.compoundAdapters[p.Type()] = append(r.compoundAdapters[p.Type()], p) + } else { + r.adapters[p.Type()] = p + } + + for _, v := range p.ConfigParameters() { + if v == "" { + continue + } + if previous, ok := r.configNameAdapterMap[v]; !ok { + r.configNameAdapterMap[v] = p.Type() + } else { + log.Printf("Duplicate config parameter: %s for %s. Previous AdapterType=%s", v, p.Type(), previous) + } + } + + scp, ok := p.(SubConfigAdapter) + if !ok { + return + } + if scp.Name() == "" { + return + } + if previous, ok := r.fileNameMap[scp.Name()]; !ok { + r.fileNameMap[scp.Name()] = p.Type() + } else { + log.Printf("Duplicate subconfig filename: %s for %s. Previous AdapterType=%s", p.Type(), scp.Name(), previous) + } +} + +func (r *AdapterRegistry) adapterForType(t AdapterType) Adapter { + vp := r.adapters[t] + if vp != nil { + return vp + } + + plugins := r.compoundAdapters[t] + if len(plugins) > 0 { + return plugins[0] + } + return nil +} + +func (r *AdapterRegistry) RemoveAdapterByType(adapterType string) error { + at := AdapterType(adapterType) + + var adapter Adapter + + nullAdapter, found := r.adapters[AdapterType("null")] + if !found { + return fmt.Errorf("%q adapter not found", "null") + } + + adapter, found = r.adapters[at] + if found { + r.adapters[at] = nullAdapter + } + + if !found { + return fmt.Errorf("unknown adapter type %q", adapterType) + } + + // override keywords + for _, v := range adapter.ConfigParameters() { + r.configNameAdapterMap[v] = nullAdapter.Type() + } + + return nil +} + +func (r *AdapterRegistry) adaptersForType(t AdapterType) []Adapter { + return r.compoundAdapters[t] +} + +func (r *AdapterRegistry) adapterTypeByConfigName(key string) AdapterType { + v, ok := r.configNameAdapterMap[key] + if !ok { + return UnknownAdapter + } + return v +} diff --git a/tools/voluspa/remap_options.go b/tools/voluspa/remap_options.go new file mode 100644 index 00000000000..e61d0c269f7 --- /dev/null +++ b/tools/voluspa/remap_options.go @@ -0,0 +1,173 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package voluspa + +import ( + "encoding/json" + "fmt" + "strconv" +) + +// RemapOptions key/value pairs from yaml configuration +type RemapOptions map[string]interface{} + +func (r RemapOptions) HasOptionSet(name string) bool { + var isSet bool + switch v := r[name].(type) { + case string: + isSet = len(v) > 0 + case bool: + isSet = v + default: + return false + } + return isSet +} + +func (r RemapOptions) Clone() RemapOptions { + ro := RemapOptions{} + for k, v := range r { + ro[k] = v + } + return ro +} + +func (r RemapOptions) HasOption(name string) bool { + _, ok := r[name] + return ok +} + +func (r RemapOptions) AddOption(name string, val interface{}) bool { + r[name] = val + return true +} + +func (r RemapOptions) RemoveOption(name string) bool { + delete(r, name) + return true +} + +func (r RemapOptions) ValueByNameAsBool(name string) (bool, error) { + v, ok := r[name].(bool) + if !ok { + return false, fmt.Errorf("value for %q is not a boolean: %v", name, r[name]) + } + return v, nil +} + +func (r RemapOptions) ValueByNameAsString(name string) (string, error) { + v, ok := r[name].(string) + if !ok { + return "", fmt.Errorf("value for %q is not a string: %v", name, r[name]) + } + return v, nil +} + +func (r RemapOptions) ValueByNameAsStringMapString(name string) (map[string]string, error) { + v, ok := r[name].(map[interface{}]interface{}) + if !ok { + return nil, fmt.Errorf("value for %q is of unexpected type, \"%T\". expected a map", name, r[name]) + } + + newv := make(map[string]string) + for k, v := range v { + newv[k.(string)] = v.(string) + } + return newv, nil +} + +func (r RemapOptions) ValueByNameAsInt(key string) (int, error) { + v, ok := r[key].(int) + if !ok { + return 0, fmt.Errorf("value for %q is not a integer: %v", key, r[key]) + } + return v, nil +} + +func (r RemapOptions) ValueByNameAsSlice(key string) ([]string, error) { + switch t := r[key].(type) { + case []interface{}: + vs := make([]string, len(t)) + for i, d := range t { + var val string + switch d := d.(type) { + case string: + val = d + case int: + val = strconv.Itoa(d) + } + + vs[i] = val + } + return vs, nil + case []string: + vs := make([]string, len(t)) + copy(vs, t) + + return vs, nil + } + return []string{}, fmt.Errorf("field %s is unhandled type: %T", key, r[key]) +} + +type remapOptionsJSON map[string]interface{} + +func (r RemapOptions) makeRemapOptionsJSONStruct() (remapOptionsJSON, error) { + rmo := remapOptionsJSON{} + for k, v := range r { + im, ok := r[k].(map[interface{}]interface{}) + if ok { + m := make(map[string]interface{}) + for ik, iv := range im { + sik, ok := ik.(string) + if !ok { + return nil, fmt.Errorf("non-string for key in map %+v", im) + } + m[sik] = iv + } + rmo[k] = m + continue + } + rmo[k] = v + } + return rmo, nil +} + +func (r RemapOptions) MarshalJSON() ([]byte, error) { + roj, err := r.makeRemapOptionsJSONStruct() + if err != nil { + return nil, err + } + return json.Marshal(roj) +} + +func (r RemapOptions) ValueByNameAsStringMapInterface(name string) (map[string]interface{}, error) { + v, ok := r[name].(map[interface{}]interface{}) + if !ok { + return nil, fmt.Errorf("value is of unexpected type. name=%s val=%v\n r=%+v\nt=%T", name, r[name], r, v) + } + + newv := make(map[string]interface{}) + for k, v := range v { + if v != nil { + newv[k.(string)] = v.(interface{}) + } + } + return newv, nil +} diff --git a/tools/voluspa/remap_options_test.go b/tools/voluspa/remap_options_test.go new file mode 100644 index 00000000000..faccbaf0d13 --- /dev/null +++ b/tools/voluspa/remap_options_test.go @@ -0,0 +1,66 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package voluspa + +import "testing" + +func TestRemapOptions_ValueByNameAsBool(t *testing.T) { + ro := &RemapOptions{} + for _, v := range []bool{true, false} { + (*ro)["key"] = v + + out, err := ro.ValueByNameAsBool("key") + if err != nil || out != v { + t.Fatalf("Expected value '%t', got '%t'", v, out) + } + } + + _, err := ro.ValueByNameAsBool("non-existent-key") + if err == nil { + t.Fatalf("Expected err getting value by non-existent key") + } + + (*ro)["key2"] = "stringval" + _, err = ro.ValueByNameAsBool("key2") + if err == nil { + t.Fatalf("Expected err getting boolean value for key") + } +} + +func TestRemapOptions_ValueByNameAsString(t *testing.T) { + ro := &RemapOptions{} + (*ro)["key"] = "value" + + out, err := ro.ValueByNameAsString("key") + if err != nil || out != "value" { + t.Fatalf("Expected value '%s', got '%s'", (*ro)["key"], out) + } + + _, err = ro.ValueByNameAsString("non-existent-key") + if err == nil { + t.Fatalf("Expected err getting value by non-existent key") + } + + (*ro)["key2"] = false + _, err = ro.ValueByNameAsString("key2") + if err == nil { + t.Fatalf("Expected err getting string value for key") + } +} diff --git a/tools/voluspa/sample.conf-template b/tools/voluspa/sample.conf-template new file mode 100644 index 00000000000..7685dd4bfba --- /dev/null +++ b/tools/voluspa/sample.conf-template @@ -0,0 +1,178 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +schema_version: '1.0' +owner: Owner Name +reference: reference number here +rcpt: MyReceipt +parent_child: true # use parent/child hierarchy +qps: 2 # Approximate queries per second to order rules +role: roles_edge_child # If this should be applied to specific salt role. +lifecycle: onboarding|live|retired #live: in production; onboarding: various things may not yet be working; retired: No longer active (no need for new ssl certs, etc); +cdn: [edge, internal] +ssl_cert_names: # a list of SSL cert filenames. used to populate ssl_multicert.config + - /secret/mysecret1.domain.com + - /secret/mysecret2.domain.com + +mappings: + - alias: + - mydomain1.com # can have multiple. If no scheme, uses from schemes: + origin: http://myorigin.domain.com + schemes: [http, https] # applied to alias and origin (if not specified) + rule: default #not needed if "default" + +rules: + default: # named set of rules to apply to this target/origin + parent_child: false # child remap goes direct to origin. (Overrides metadata's parent_child for this ruleset) + strip_query: true # strips the query string completely -- the origin won't see it. + force_ttl: 7d # Always set Cache-Control: max-age: $time; overwrites if origin had set. + default_ttl: 7d # If the Origin doesn't set a Cache-Control, set it to Cache-Control: max-age: $time + negative_ttl: 10s # amount of time to cache http errors (404, 502, 504, etc) + transaction_timeout: 10m # Sets transaction timeouts (CDN default is 30s) + content_type_forge: # Change the content type based on a series of regexes (Need this line too) + # 'regex': content-type. urls matching regex will have have the Content-type header set to content-type + '.*\.json$': application/json + '.*\.css$': text/css + edge_compression: # if the origin does not surpport compression, CDN should do the compression instead + enabled: true + algorithms: [gzip] # only gzip for now + static_origin_compressible_content_type: + - application/json #compress only these mime types -- not the defaults + header_rewrite: |- # raw values to put into header_rewrite configration + cond %{SEND_REQUEST_HDR_HOOK} + set-header Authorization "Basic 123412341234" + deny_methods: [ CONNECT, POST, PUT, DELETE ] #Reject these HTTP verbs + priority: background | foreground | streamingaudio | streamingvideo # QOS priority (DSCP bits) + set_header: 'X-Client-Protocol "%"'# Set this header; overwrites if origin had set. + add_header: 'X-Client-Protocol "%"' # add this header; + remove_header: "X-Client-Protocol" # remove this header; + add_header_origin: 'X-Client-Protocol "%"' # Send this header to the origin + set_header_origin: 'X-Client-Protocol "%"' # Send this header to the origin; overwrites if client had set. + proxy_cache_control: "MyProxyHeader" # Use this header's value for Cache-Control. + disable_cache: true # turn off the cache for this rule + regex_remap: "^/([^/]+)/(.*) https://$1.s3.amazonaws.com/$2" # use regex to match the inbound path (only) and substition for origin. Can add optional http status code (like @status=301). Can also use "|-" format + conf_remap: |- # raw values to override specific ATS settings. + CONFIG proxy.config.http.connect_attempts_timeout INT 600 + CONFIG proxy.config.http.negative_caching_enabled FLOAT .050 + allow_ip: [10.0.0.0-10.0.255.255, 10.13.0.0-10.13.255.255] # Only accept requests from these IP ranges + log_cookie: # log specific cookies for this property. Logger-cookie-Name1 will become the key name in logger + Logger-cookie-Name1: NameOfCookie1 + Logger-cookie-Name2: NameOfCookie2 + log_header: # log specific header for this property. Logger-header-Name1 will become the key name in logger + Logger-header-Name1: NameOfHeader1 + Logger-header-Name2: NameOfHeader2 + log_type: private|public # Private logs no personal info (no client IP, url, referalurl ...); Public is default + cachekey: #modify the cache key + include_params: ["param1", "param2"] # only use these query parameters in the cachekey + exclude_params: ["param1", "param2"] # ignore these query parameters in the cachekey + sort_query: true # sort the query parameters for the cachekey + remove_all_params: true # Don't use the query string as part of the cachekey. + include_headers: ["Origin", "Content-Type"] # include specific headers in the cachekey + include_cookies: ["Cookie1", "Cookie2"] # include specific cookies in the cachekey + static_prefix: SomeString # specified value will be added to the cachekey + regex_replace_path: '/.*/filename.html/' # replace cachekey URI path with regex capture + cache_promote: # cache promotion plugin options -- only cache the popular objects + policy: lru | chance + sample: 25 + lru_hits: 10 + lru_buckets: 10000 + failover: # escalate plugin options -- when object is unavailable, try another origin + domain: domain.com + status_codes: [401, 403, 404, 407, 410, 500, 501, 502, 503, 504, 505] + host_header: alias|origin # What hostname should we use when contacting the failover? (alias or origin) +# Flags to disable options automatically enabled: + receipts: + enabled: false # don't set a rcpt field in logs + name: override_name # override receipt name on ruleset + propstats: false # don't send stats to epic for this remap rule + origin_host_header: alias| origin # origin=Send the origin's Host: header to the origin. alias=send the inbound Host: header to origin (default)" +# Property Specific options: + s3: #Use Amazon S3 authentication + path: "path/to/config" # Origin is s3; eg certs/s3.config + version: 2|4 # which version of Amazon AWS signature algorithm to use; 2 or 4 with default of 2 + virtual_host: true # populates Host: header with S3 virtual host + authproxy: range | head # Send Range/Head request to secondary origin to authenticate request. Will need 2nd remap rule to catch these requests. +# Video background fetch plugin options + video_background_fetch: + fetch_count: 10 + fetch_max: 1000 + api_header: header1 + replace_host: host1 + name_space: ns1 + metrics_prefix: prefix1 + exact_match: true + log_name: log1 + frontend: + fetch_path_pattern: '/(.*-)(\d+)(.*)/$1{$2+2}$3/' + fetch_policy: lru:10000000 + backend: + fetch_policy: simple + +tests: +# Status code checks + health: # arbitrary name for test + urls: # urls to test + - http://mydomain1.com/show/health + - https://mydomain1.com/show/health + success: # rules defining a successful test + status_code: 200 + +# Header checks + parent_cache_key_and_etag: + description: run only on hosts in trafficserver_parent salt role + role: roles_uat # specify role of host to run test against + insecure: true # Default False. Ignore ssl errors thus allowing insecure connection + range: 0-499 # specifies the first 500 bytes + purge_before: true # Default False. Purge url before running a test + headers: # request headers + X-Debug: X-Cache-Key + host_types: # valid values: parent|child + - parent + urls: + - http://mydomain1.com/test.txt + success: + status_code: 200 + headers: + X-Cache-Key: /mydomain1.com:80/test.txt + ETag: \"d36f8f9425c4a8000ad9c4a97185aca5\" +# domain names for SSL tests + ssl_tests: + domain_names: + - working.domain.com + + cert_serial_number_check: + description: devimages has a pinned cert + type: ssl + domain_names: + - devimages-cdn.domain.com + success: + serial_numbers: + - ABC123 + +# log settings +receiptsd: # configure receipts (logs) sent + sampling: + posters: + global: 10% # only send 10% of logs to main instance + adcdownload: 100% # also send 100% of logs to the download instance + ignored_extensions: # don't send any logs for files with these extensions + - .dist + - .dist.gz + - .pkm + +# vim: set et sw=2 ts=2 ft=yaml : diff --git a/tools/voluspa/schema_v1.json b/tools/voluspa/schema_v1.json new file mode 100644 index 00000000000..2abbcc0235d --- /dev/null +++ b/tools/voluspa/schema_v1.json @@ -0,0 +1,1126 @@ +{ + "$schema": "http://json-schema.org/schema#", + "$comment": "Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements; and to You under the Apache License, Version 2.0.", + "type": "object", + "additionalProperties": false, + "required": [ + "rcpt", + "schema_version", + "owner" + ], + "definitions": { + "schemes": { + "type": "array", + "uniqueItems": true, + "items": { + "type": "string", + "enum": [ + "http", + "https" + ], + "minItems": 1 + }, + "description": "http|https applied to alias and origin (if not specified there)" + }, + "role": { + "type": "string", + "enum": [ + "roles_edge_child" + ], + "description": "If this should be applied to specific set of machines." + }, + "parent_child": { + "type": "boolean", + "description": "Use parent/child hierarchy" + }, + "alias": { + "type": "array", + "items": { + "type": "string" + }, + "minItems": 1, + "description": "Inbound URI. Can have multiple. If there is no scheme, uses from schemes:. Any extra path and query parameters are passed along at end of origin." + }, + "origin": { + "type": "string", + "description": "Outbound URL. If there is no scheme, uses it from schemes:. Extra path and query parameters after aliases are passed along at the end." + }, + "propstats": { + "type": "boolean", + "description": "Keep metrics on this property or not." + }, + "fetch_policy": { + "type": "string", + "description": "simple or lru (with lru size). eg lru:10000000" + }, + "mapping": { + "type": "object", + "additionalProperties": false, + "required": [ + "alias", + "origin" + ], + "items": { + "type": "object" + }, + "properties": { + "comment": { + "type": "string" + }, + "alias": { + "$ref": "#/definitions/alias" + }, + "origin":{ + "$ref": "#/definitions/origin" + }, + "group": { + "type": "string", + "enum": [ + "group1", + "group2", + "group3", + "group4" + ], + "description": "Used to override resolver." + }, + "rule": { + "type": "string", + "description": "Which ruleset to apply to this mapping set" + }, + "parent_dest_domain": { + "type": "string", + "description": "May need to tell Voluspa what domain to use when finding the parent. Note: this option is being deprecated! Use parent_override instead!" + }, + "regex_map": { + "type": "boolean", + "description": "if there is a regular expression in the hostname of this uri." + }, + "role": { + "$ref": "#/definitions/role" + }, + "schemes": { + "$ref": "#/definitions/schemes" + }, + "parent_override": { + "description": "List of overrides for parent.config.", + "type": "array", + "items": { + "type": "object", + "additionalProperties": false, + "properties": { + "comment": { + "type": "string" + }, + "role": { + "$ref": "#/definitions/role" + }, + "dest_domain": { + "type": "string", + "description": "What destination domain when finding the parent." + }, + "strategy": { + "type": "string", + "enum": [ + "true_round_robin", + "strict_round_robin", + "no_round_robin", + "consistent_hash", + "latched" + ], + "default": "consistent_hash", + "description": "Routing strategy." + }, + "ignore_querystring": { + "type": "boolean", + "default": false, + "description": "When set to true the query string will not be included in the hash to select the parent." + }, + "http_port": { + "type": "integer", + "description": "Port to use for http scheme." + }, + "https_port": { + "type": "integer", + "description": "Port to use for https scheme." + }, + "go_direct": { + "type":"boolean", + "default": false, + "description": "When set to true, requests bypass parent hierarchies and go directly to the origin server." + }, + "primary_list": { + "type":"array", + "minItems": 1, + "items": { + "anyOf": [ + { + "type": "string", + "format": "hostname" + }, + { + "type": "string", + "format":"ipv4" + }, + { + "type": "string", + "format":"ipv6" + } + ] + }, + "description": "Array of hostnames (or IP adderesses) of the primary parents." + }, + "secondary_list": { + "type":"array", + "minItems": 1, + "items": { + "anyOf": [ + { + "type": "string", + "format": "hostname" + }, + { + "type": "string", + "format":"ipv4" + }, + { + "type": "string", + "format":"ipv6" + } + ] + }, + "description": "Array of hostnames (or IP adderesses) of the secondary parents." + } + } + } + } + } + } + }, + "properties": { + "comment": { + "type": "string" + }, + "schema_version": { + "oneOf": [ + { "type": "string", "pattern": "^1(\\.0)?$" }, + { "type": "integer", "minimum": 1, "maximum": 1 } + ], + "description": "Declares the version of the Voluspa schema this configuration is defined in terms of." + }, + "alias": { + "$ref": "#/definitions/alias" + }, + "cdn": { + "type": "array", + "minItems": 1, + "uniqueItems": true, + "items": { + "type": "string", + "enum": [ + "edge", + "internal" + ] + }, + "description": "Which machines to target." + }, + "lifecycle": { + "type": "string", + "enum": [ + "onboarding", + "live", + "retired" + ], + "description": "live: in production; onboarding: various things may not yet be working; retired: No longer active (no need for new ssl certs, etc);" + }, + "org": { + "type": "string", + "enum": [ + "IST", + "InternetServices", + "SoftwareEngineering" + ], + "description": "Used for billing the correct org. Metadata only, not used for cache configuration." + }, + "owner": { + "type": "string", + "description": "What group should we email/page if there are problems?" + }, + "propstats": { + "$ref": "#/definitions/propstats" + }, + "rcpt": { + "type": "string", + "description": "A unique property name." + }, + "reference": { + "type": "string", + "description": "Where to find more information about this property, such as a URL." + }, + "role": { + "$ref": "#/definitions/role" + }, + "rules": { + "type": "object", + "additionalProperties": false, + "patternProperties": { + ".*": { + "type": "object", + "additionalProperties": false, + "properties": { + "comment": { + "type": "string" + }, + "set_header": { + "type": "string", + "description": "Set this header; overwrites if origin had set." + }, + "cachekey": { + "type": "object", + "additionalProperties": false, + "items": { + "type": "string" + }, + "description": "Defines how the cachekey is composed. Changing any of these parameters is likely to effectively clear the cache.", + "properties": { + "comment": { + "type": "string" + }, + "remove_all_params": { + "type": "boolean", + "description": "Don't use the query string as part of the cachekey or parent_selection." + }, + "include_headers": { + "type": "array", + "items": { + "type": "string" + }, + "description": "Include specific headers from the request in the cachekey." + }, + "regex_replace_path": { + "type": "string", + "examples": [ + "/.*/probe-info-ok.html/" + ], + "description": "Replace cachekey URI path with regex capture. Format is PCRE pattern or PCRE pattern + replacement in format '///'" + }, + "regex_replace_path_uri": { + "type": "string", + "examples": [ + ["/http[s]?:\\/\\/[^\\/]*(.*\\.html|.*).*/$1/"] + ], + "description": "Replace cachekey full URI with regex capture. Format is PCRE pattern or PCRE pattern + replacement in format '///'" + }, + "include_params": { + "type": "array", + "items": { + "type": "string" + }, + "examples": [ + ["queryparam1", "queryparam2"] + ], + "description": "Only use these query parameters in the cachekey." + }, + "static_prefix": { + "type": "string", + "description": "Specified value will start the cachekey." + }, + "include_cookies": { + "type": "array", + "uniqueItems": true, + "items": { + "type": "string" + }, + "description": "Include these specific cookies in the cachekey." + }, + "exclude_params": { + "type": "array", + "uniqueItems": true, + "items": { + "type": "string" + }, + "examples": [ + ["queryparam1", "queryparam2"] + ], + "description": "Ignore these query parameters in the cachekey." + }, + "sort_query": { + "type": "boolean", + "description": "Sort the query parameters for the cachekey." + }, + "parent_selection": { + "type": "string", + "enum": [ + "ignore_query" + ], + "description": "ignore_query: Ignore the query string when choosing the parent. Note: this option is being deprecated! Use parent_override in stead!" + }, + "capture_header": { + "type": "array", + "uniqueItems": true, + "items": { + "type": "string" + }, + "examples": [ + [ "Authorization:/AWS\\s([^:]+).*/clientID:$1/", "Authorization:/BYT\\s([^,]+).*/clientID:$1/" ] + ], + "description": "Add specific parts of headers to the cachekey via a regex capture. Format is PCRE pattern or PCRE pattern + replacement in format '///'" + } + } + }, + "allow_ip": { + "type": "array", + "uniqueItems": true, + "items": { + "type": "string" + }, + "description": "Only accept requests from these IPs and IP ranges.", + "examples": [ + [ "127.0.0.1", "::1" ], + [ "10.0.0.0-10.255.255.255", "127.0.0.1", "::1" ] + ] + }, + "allow_ip_parent": { + "type": "array", + "uniqueItems": true, + "items": { + "type": "string" + }, + "description": "Only accept requests from these IPs and IP ranges (applies to parent hosts).", + "examples": [ + [ "10.0.0.0-10.255.255.255", "10.127.0.0-10.127.255.255" ] + ] + + }, + "header_rewrite": { + "type": "string", + "description": "Raw text to put into ATS header_rewrite configuration." + }, + "add_header": { + "type": "string", + "description": "Add this header to the response.", + "examples": [ "Strict-Transport-Security \"max-age=31536000; includeSubDomains\"" ] + }, + "add_header_origin": { + "type": "string", + "description": "Add this header to the origin request.", + "examples": [ "X-CLIENT-IP \"%\"" ] + }, + "origin_host_header": { + "type": "string", + "enum": [ + "origin", + "alias" + ], + "description": "What Host: to send to the origin. origin=Send the origin's Host; alias=send the inbound Host: (default)" + }, + "content_type_forge": { + "type": "object", + "patternProperties": { + ".*": { + "type": "string", + "description": "'URL regex': content-type. URLs matching regex will have have the response Content-Type set to content-type" + } + }, + "description": "Change the Content-Type based on a series of URL path regexes", + "examples": [ "\".*\\.json$\": \"application/json\"" ] + }, + "conf_remap": { + "type": "string", + "description": "Raw text to put into ATS conf_remap configuration." + }, + "cache_promote": { + "type": "object", + "additionalProperties": false, + "description": "Configuration for the cache_promote plugin. Selectively cache objects. For really large working sets.", + "properties": { + "comment": { + "type": "string" + }, + "policy": { + "type": "string", + "enum": [ + "lru", + "chance" + ], + "description": "`chance`: randomly choose to cache the item (based on sample number); `lru`: Track most recently used URLs, and cache those with more than lru_hits." + }, + "sample": { + "type": "number", + "minimum": 1, + "maximum": 100, + "description": "The percentage sampling rate for the request to be considered, for either policy." + }, + "lru_hits": { + "type": "integer", + "description": "In `lru` policy, how often the object is fetched before promotion." + }, + "lru_buckets": { + "type": "integer", + "description": "In `lru` policy, how many objects are tracked." + } + } + }, + "propstats": { + "$ref": "#/definitions/propstats" + }, + "regex_remap": { + "type": "string", + "description": "Use regex to match the inbound path (only) and substition for origin. Can add optional http status code (like @status=301). Can also use '|-' format", + "examples": [ "^/([^/]+)/([^\\?]*)\\??.* https://$1.s3.amazonaws.com/$2" ] + }, + "negative_ttl": { + "type": "string", + "description": "Amount of time to cache http errors (404, 502, 504, etc). Default 10s.", + "examples": [ "10s" ] + }, + "priority": { + "type": "string", + "enum": [ + "background", + "foreground", + "streamingaudio", + "streamingvideo" + ], + "description": "QOS priority (DSCP bits)" + }, + "default_ttl": { + "type": "string", + "description": "If the Origin doesn't set a Cache-Control, set it to this value. A number followed by a letter (like 17s, 5m, 3h, 2d, 1w) is a shortcut for 'max-age=N, public'", + "examples": [ "10s", "60m", "12h", "no-cache, no-store, must-revalidate, stale-while-revalidate=30" ], + "pattern": "^([0-9]+[smhdw])$|^((no-cache|no-store|no-transform|must-revalidate|public|private|proxy-revalidate|immutable|(max-age=|s-maxage=|stale-while-revalidate=|stale-if-error=)[0-9]+) *(, *|$))+$" + }, + "set_header_origin": { + "type": "string", + "description": "Send this header to the origin; overwrites if client had set." + }, + "authproxy": { + "type": "string", + "emum": [ + "range", + "head" + ], + "description": "Send Range/Head request to secondary origin to authenticate request. Will need 2nd remap rule to catch these requests." + }, + "parent_child": { + "$ref": "#/definitions/parent_child" + }, + "parent": { + "type": "boolean" + }, + "child": { + "type": "boolean" + }, + "redirect": { + "type": "object", + "additionalProperties": false, + "description": "Configure the Regex Remap Plugin. See https://docs.trafficserver.apache.org/en/7.1.x/admin-guide/plugins/regex_remap.en.html", + "properties": { + "comment": { + "type": "string" + }, + "src": { + "type": "string", + "description": "regex src pattern, defaults to `(.*)`" + }, + "url": { + "type": "string", + "description": "regex URL to redirect to, may include substitution variables" + }, + "http_code": { + "type": "integer", + "minimum": 100, + "maximum": 999, + "description": "HTTP status code to return, defaults to 302" + } + }, + "examples": [ + { "url": "https://example.com$0", "http_code": 410 }, + { "src": "http://old.(.*).z.com", "url": "http://new.$1.z.com" } + ] + }, + "video_background_fetch": { + "type": "object", + "description": "Prefetch sequential URLs for improved caching. Usually for HLS video.", + "additionalProperties": false, + "properties": { + "comment": { + "type": "string" + }, + "log_name": { + "type": "string", + "description": "Rcpt to log these requests." + }, + "fetch_count": { + "type": "integer", + "description": "How many URLs ahead to fetch." + }, + "replace_host": { + "type": "string", + "description": "What Host: header to use in pre-fetch." + }, + "exact_match": { + "type": "boolean" + }, + "api_header": { + "type": "string", + "description": "What header to use to signal this is a prefetch request." + }, + "name_space": { + "type": "string", + "description": "Name Space to segregate LRU count." + }, + "metrics_prefix": { + "type": "string", + "description": "Prefix for prefetch metrics." + }, + "fetch_policy": { + "$ref": "#/definitions/fetch_policy" + }, + "frontend": { + "type": "object", + "properties": { + "comment": { + "type": "string" + }, + "fetch_policy": { + "$ref": "#/definitions/fetch_policy" + }, + "fetch_path_pattern": { + "type": "array", + "minItems": 1, + "uniqueItems": true, + "items": { + "type": "string" + }, + "description": "Regex to find the portion of the url that can be incremented." + } + } + }, + "fetch_max": { + "type": "integer" + }, + "backend": { + "type": "object", + "properties": { + "comment": { + "type": "string" + }, + "fetch_policy": { + "type": "string", + "description": "simple or lru (with lru size). eg lru:10000000", + "pattern": "^(simple|lru:[0-9]+)$" + } + } + } + } + }, + "log_header": { + "type": "object", + "properties": {}, + "description": "Log specific request headers with the specified keyword names.", + "examples": [ { "name-in-log": "name-of-header", "cacheControlValue": "Cache-Control", "playbackSessionID": "X-PlayBack-Session-Id" } ] + }, + "proxy_cache_control": { + "type": "string", + "description": "Use this header's value for Cache-Control.", + "examples": [ "Proxy-Cache-Control" ] + }, + "failover": { + "type": "object", + "description": "When object is unavailable, try a different origin.", + "additionalProperties": false, + "properties": { + "comment": { + "type": "string" + }, + "domain": { + "type": "string", + "description": "When object is unavailable, try what origin?" + }, + "status_codes": { + "type": "array", + "uniqueItems": true, + "items": { + "type": "integer", + "minimum": 100, + "maximum": 999 + }, + "description": "What HTTP status codes indicate we should try another origin?" + }, + "host_header": { + "type": "string", + "enum": [ + "origin", + "alias" + ], + "description": "What hostname should we use when contacting the failover? (alias or origin)" + } + } + }, + "remove_header": { + "type": "string", + "description": "Remove this header." + }, + "remove_header_origin": { + "type": "string", + "description": "Remove this header in origin request." + }, + "log_cookie": { + "type": "object", + "properties": {}, + "description": "Log specific request cookies with the specified keyword names.", + "examples": [ { "name-in-log": "name-of-cookie", "session": "s", "auth": "ADCDownloadAuth" } ] + }, + "force_ttl": { + "type": "string", + "description": "Always set Cache-Control to this value; overwrites if origin had set. A number followed by a letter (like 17s, 5m, 3h, 2d, 1w) is a shortcut for 'max-age=N, public'", + "pattern": "^([0-9]+[smhdw])$|^((no-cache|no-store|no-transform|must-revalidate|public|private|proxy-revalidate|immutable|(max-age=|s-maxage=|stale-while-revalidate=|stale-if-error=)[0-9]+) *(, *|$))+$" + }, + "receipts": { + "type": "object", + "description": "Control receipt (eg log) production.", + "additionalProperties": false, + "properties": { + "comment": { + "type": "string" + }, + "enabled": { + "type": "boolean", + "description": "Should we send receipts (eg logs) for this property?" + }, + "name": { + "type": "string", + "description": "Should we use a different rcpt name for this ruleset?" + } + } + }, + "allow_methods": { + "type": "array", + "uniqueItems": true, + "items": { + "type": "string", + "enum": [ + "GET", + "HEAD", + "POST", + "PUT", + "DELETE", + "CONNECT", + "OPTIONS", + "TRACE", + "PATCH", + "PURGE" + ] + }, + "description": "Accept only these HTTP verbs" + }, + "deny_methods": { + "type": "array", + "uniqueItems": true, + "items": { + "type": "string", + "enum": [ + "GET", + "HEAD", + "POST", + "PUT", + "DELETE", + "CONNECT", + "OPTIONS", + "TRACE", + "PATCH" + ] + }, + "description": "Reject these HTTP verbs" + }, + "strip_query": { + "type": "boolean", + "description": "Strips the query string completely -- the origin won't see it." + }, + "transaction_timeout": { + "type": "string", + "description": "Sets transaction timeouts (CDN default is 60s)" + }, + "s3": { + "type": "object", + "additionalProperties": false, + "description": "Use Amazon S3 authentication when contacting origin.", + "required": [ + "path" + ], + "properties": { + "comment": { + "type": "string" + }, + "path": { + "type": "string", + "description": "Path to the s3 configuration containing the shared secret." + }, + "virtual_host": { + "type": "boolean", + "description": "Populate Host: header with S3 virtual host." + }, + "version": { + "type": "integer", + "enum": [ + 2, + 4 + ], + "description": "Which version of Amazon AWS signature algorithm to use." + }, + "v4_include_headers": { + "type": "array", + "items": { + "type": "string" + } + }, + "v4_exclude_headers": { + "type": "array", + "items": { + "type": "string" + }, + "description": "Don't use these headers in the authorization signature.", + "examples": [ { "v4_exclude_headers": ["x-forwarded-for","forwarded","via","authorization"] } ] + }, + "region_map": { + "type": "string", + "description": "Path to a region map config file." + } + } + }, + "edge_compression": { + "type": "object", + "additionalProperties": false, + "description": "", + "properties": { + "comment": { + "type": "string" + }, + "enabled": { + "type": "boolean", + "description": "Should we enable compression?" + }, + "algorithms": { + "type": "array", + "uniqueItems": true, + "items": { + "type": "string", + "enum": [ + "gzip", + "br" + ], + "description": "Specify compression algorithm(s) to use." + } + }, + "static_origin_compressible_content_type": { + "type": "array", + "uniqueItems": true, + "items": { + "type": "string" + }, + "description": "Compression for specified mime-types.", + "examples": [ [ "application/x-mpegURL" ] ] + }, + "flush": { + "type": "boolean" + }, + "remove_accept_encoding": { + "type": "boolean", + "description": "Removes Accept-Encoding header in origin requests." + }, + "minimum_content_length": { + "type": "integer", + "description": "Minimum content length for compression to be enabled (in bytes)" + } + } + }, + "alt_svc": { + "type": "object", + "additionalProperties": false, + "properties": { + "comment": { + "type": "string" + }, + "authority": { + "type": "string" + }, + "protocol": { + "type": "string" + }, + "ma": { + "type": "integer" + }, + "health": { + "type": "string" + }, + "parameters": { + "type": "array", + "items": { + "type": "string" + } + } + } + }, + "disable_cache": { + "type": "boolean", + "description": "Turn off the cache for this rule." + }, + "log_type": { + "type": "string", + "enum": [ + "private", + "public" + ], + "description": "Private if we should not log privacy sensitive information, like the URL, referral URL, client IP, etc. Default is public." + }, + "access_approval": { + "type": "object", + "additionalProperties": false, + "properties": { + "comment": { + "type": "string" + }, + "raw": { + "type": "string" + } + } + }, + "storage_volume": { + "type": "string", + "description": "Selects which volume will be used for caching (disk_volume or ramdisk_volume); when this is omitted the default used is disk_volume.", + "enum": [ + "disk_volume", + "ramdisk_volume" + ] + }, + "lua": { + "type": "string", + "description": "Raw text to run as Lua remap script." + }, + "echo_cors": { + "type": "string", + "description": "Header to echo back as Access-Control-Allow-Origin in response." + }, + "cache_version": { + "type": "integer", + "description": "To clear the cache for this rule, increment the integer (or add this rule with value 1)." + } + } + } + } + }, + "schemes": { + "$ref": "#/definitions/schemes" + }, + "mappings": { + "type": "array", + "items": { + "$ref": "#/definitions/mapping" + } + }, + "explicit": { + "type": "array", + "items": { + "$ref": "#/definitions/mapping" + }, + "uniqueItems": true + }, + "ha_proxy": { + "type": "array", + "additionalProperties": false, + "items": { + "type": "object", + "required": [ + "port", + "rips" + ] + }, + "properties": { + "comment": { + "type": "string" + }, + "port": { + "type": "integer" + }, + "rips": { + "type": "array", + "items": { + "type": "object", + "required": [ + "ip", + "port" + ] + }, + "properties": { + "ip": { + "type": "string" + }, + "port": { + "type": "integer" + } + } + } + } + }, + "parent_child": { + "$ref": "#/definitions/parent_child" + }, + "qps": { + "type": "integer", + "minimum": 1, + "description": "Approximate queries per second, used to order rules for efficiency." + }, + "receiptsd": { + "type": "object", + "properties": { + "comment": { + "type": "string" + }, + "ignored_extensions": { + "type": "array", + "items": { + "type": "string" + } + }, + "sampling": { + "type": "object", + "properties": { + "comment": { + "type": "string" + }, + "posters": { + "type": "object", + "description": "Name of the program posting the logs. Current values are Global (for iTunes splunk), adcdownload, and adcdownload-uat (for posting to WWDR's prod/uat)", + "examples": [ "global: 1%" ], + "patternProperties": { + ".*": { + "type": "string", + "pattern": "^\\d+%$" + } + } + } + }, + "additionalProperties": false + } + } + }, + "ssl_cert_names": { + "type": "array", + "uniqueItems": true, + "items": { + "type": "string", + "pattern": "^([\\w\\-\\.,:/]+)$" + }, + "description": "A list of SSL cert filenames. used to populate ssl_multicert.config." + }, + "tests": { + "type": "object", + "additionalProperties": false, + "patternProperties": { + ".*": { + "type": "object", + "properties": { + "comment": { + "type": "string" + }, + "description": { + "type": [ + "integer", + "string" + ] + }, + "domain_names": { + "type": "array", + "minItems": 1, + "uniqueItems": true, + "items": { + "type": "string" + }, + "description": "Domain names for SSL tests" + }, + "headers": { + "type": "object", + "patternProperties": { + ".*": { + "type": "string" + } + }, + "description": "Headers to go into the request." + }, + "host_types": { + "type": "array", + "minItems": 1, + "uniqueItems": true, + "items": { + "type": "string", + "enum": [ + "parent", + "child" + ] + }, + "description": "Which kind of hosts should the check run on?" + }, + "insecure": { + "type": "boolean", + "description": "If curl should ignore certificate warnings." + }, + "purge_before": { + "type": "boolean", + "description": "If we should run a PURGE command before the test." + }, + "range": { + "type": "string", + "description": "Use a range request -- eg, 0-499 will ask for the 1st 500 bytes." + }, + "role": { + "type": "string", + "description": "Specify role of host to run test against." + }, + "success": { + "type": "object", + "additionalProperties": false, + "properties": { + "comment": { + "type": "string" + }, + "headers": { + "type": "object", + "patternProperties": { + ".*": { + "type": "string" + } + }, + "description": "Headers and their values that should be returned" + }, + "serial_numbers": { + "type": "array", + "uniqueItems": true, + "minItems": 1, + "items": { + "type": "string" + }, + "description": "Serial Number of cert or Intermediate." + }, + "status_code": { + "type": "integer", + "minimum": 100, + "maximum": 999, + "description": "HTTP status code expected in the response." + } + } + }, + "urls": { + "type": "array", + "minItems": 1, + "uniqueItems": true, + "items": { + "type": "string" + }, + "description": "URLs to test" + } + } + } + } + } + } +} diff --git a/tools/voluspa/specs/voluspa.spec b/tools/voluspa/specs/voluspa.spec new file mode 100644 index 00000000000..75c85f6a24a --- /dev/null +++ b/tools/voluspa/specs/voluspa.spec @@ -0,0 +1,53 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +%{!?_release: %define _release 0} +%{!?_version: %define _version UNKNOWN} +%{!?commit0: %define commit0 UNKNOWN} + +Name: voluspa +Version: %{_version} +Release: 1 +Summary: Voluspa +License: Apache Software License 2.0 (AL2) +Group: Applications/System +URL: https://github.com/apache/trafficserver/tools/voluspa + +%description +CDN -> ATS Configuration Tool + +Version: %{commit0} + +%prep + +%build + +%check + +%install +rm -rf $RPM_BUILD_ROOT +mkdir -p $RPM_BUILD_ROOT/usr/bin +cp -p %{_topdir}/dist/linux/voluspa $RPM_BUILD_ROOT/usr/bin + +%files +%defattr(-, root, root, -) +/usr/bin/voluspa + +%clean + +%changelog diff --git a/tools/voluspa/ssl_multicert.go b/tools/voluspa/ssl_multicert.go new file mode 100644 index 00000000000..cb4e491244d --- /dev/null +++ b/tools/voluspa/ssl_multicert.go @@ -0,0 +1,199 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package voluspa + +import ( + "bytes" + "fmt" + "sort" + "strings" +) + +var ( + // defaultCertNames is a set of cert names to be specified as the default + defaultCertNames = map[string]interface{}{ + "/secret/edgecdn:images.domain.com": nil, + "/secret/images.domain.com.rsa": nil, + "/secret/images.domain.com.rsa,/secret/images.domain.com.ecdsa": nil, + } + + SSLMulticertDefaultFilename = "ssl_multicert.config_default" + SSLMulticertFilename = "ssl_multicert.config" +) + +type sslCert struct { + filename string + comment string +} + +type SSLMulticertConfigurator struct { + options *Options +} + +func newSSLMulticertConfigurator(options *Options) *SSLMulticertConfigurator { + return &SSLMulticertConfigurator{options: options} +} + +func initCertsMap(parsedConfigs []*CustomerConfig) map[string]map[string]sslCert { + // a map of roles to an array of sslCerts + certs := make(map[string]map[string]sslCert) + + sort.Stable(sort.Reverse(byQPSAndName(parsedConfigs))) + + for _, parsedConfig := range parsedConfigs { + if len(parsedConfig.sslCertNames) == 0 { + continue + } + if parsedConfig.lifecycle == Retired { + continue + } + + role := parsedConfig.role + + if _, exists := certs[role]; !exists { + certs[role] = make(map[string]sslCert) + } + + sort.Strings(parsedConfig.sslCertNames) + + for _, sslCertName := range parsedConfig.sslCertNames { + certs[role][sslCertName] = sslCert{ + filename: sslCertName, + comment: parsedConfig.reference, + } + } + } + + return certs +} + +func (s *SSLMulticertConfigurator) Do(parsedConfigs []*CustomerConfig, merge bool) ([]ManagedFile, error) { + if merge { + return s.get(parsedConfigs, NoCDN) + } + + // otherwise, group properties by CDN and generate ssl_multicert.config for each CDN + grouped := groupCustomerConfigsByCDN(parsedConfigs) + var managedFiles []ManagedFile + for cdn, configs := range grouped { + files, err := s.get(configs, cdn) + if err != nil { + return nil, err + } + managedFiles = append(managedFiles, files...) + } + + return managedFiles, nil +} + +func (s *SSLMulticertConfigurator) get(configs []*CustomerConfig, cdn string) ([]ManagedFile, error) { + certs := initCertsMap(configs) + if len(certs) == 0 { + return nil, nil + } + + var buf bytes.Buffer + buf.WriteString(generatedFileBanner) + var roles []string + for role := range certs { + roles = append(roles, role) + } + sort.Strings(roles) + + for _, role := range roles { + err := s.expandConfigTemplate(certs, role, &buf) + if err != nil { + return nil, err + } + } + + fileName := SSLMulticertDefaultFilename + if len(cdn) > 0 && (cdn != DefaultCDN && cdn != NoCDN) { + fileName = fmt.Sprintf("ssl_multicert.config_%s", cdn) + } + + return []ManagedFile{ + NewManagedFile(fileName, SSLMulticertFilename, cdn, "", "", &buf, UnknownLocation), + }, nil +} + +func (s *SSLMulticertConfigurator) startRoleGuard(role string) string { + if len(role) == 0 { + if s.options.PromoteRolesToCDN { + return "" + } + + return "{% if not salt.pillar.get('roles_uat') %}\n\n" + } + + roleName := role + if !strings.HasPrefix(role, "roles_") { + roleName = fmt.Sprintf("roles_%s", role) + } + return fmt.Sprintf("{%% if salt.pillar.get('%s') %%}\n\n", roleName) +} + +func (s *SSLMulticertConfigurator) endRoleGuard(role string) string { + if len(role) == 0 && s.options.PromoteRolesToCDN { + return "" + } + return "{% endif %}\n\n" +} + +func (s *SSLMulticertConfigurator) expandConfigTemplate(certsMap map[string]map[string]sslCert, role string, buf *bytes.Buffer) error { + certs, ok := certsMap[role] + if !ok { + return fmt.Errorf("role %q not found", role) + } + + if len(certs) == 0 { + return nil + } + + var certFilenames []string + for certFilename := range certs { + certFilenames = append(certFilenames, certFilename) + } + sort.Strings(certFilenames) + + buf.WriteString(s.startRoleGuard(role)) + + var sawDefaultCert bool + for _, certFilename := range certFilenames { + cert := certs[certFilename] + if len(cert.comment) > 0 { + buf.WriteString(fmt.Sprintf("# %s\n", cert.comment)) + } + if _, found := defaultCertNames[certFilename]; found { + if sawDefaultCert { + return fmt.Errorf("more than one default cert found for role") + } + + buf.WriteString("dest_ip=* ") + sawDefaultCert = true + } + + buf.WriteString(fmt.Sprintf("ssl_cert_name=%s\n\n", certFilename)) + } + + buf.WriteString(s.endRoleGuard(role)) + + return nil +} diff --git a/tools/voluspa/tests/.gitignore b/tools/voluspa/tests/.gitignore new file mode 100644 index 00000000000..b98b0e32b34 --- /dev/null +++ b/tools/voluspa/tests/.gitignore @@ -0,0 +1,5 @@ +*/test +*/*/test +/out +*/out +*.out diff --git a/tools/voluspa/tests/Makefile b/tools/voluspa/tests/Makefile new file mode 100644 index 00000000000..dbe8b164d72 --- /dev/null +++ b/tools/voluspa/tests/Makefile @@ -0,0 +1,53 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +test: regress other custom + +clean: + @rm -f */system.json + @rm -f */profile_*.out + +COVERAGE_OPTIONS=-test.run ^TestRunMain$$ -test.coverprofile=system.out + +coverage: + cd regress && OPTIONS="$(COVERAGE_OPTIONS)" VOLUSPA=voluspa-covered ./test.sh + cd other && OPTIONS="$(COVERAGE_OPTIONS)" VOLUSPA=voluspa-covered ./test.sh + if [ -x custom ]; then \ + cd custom && OPTIONS="$(COVERAGE_OPTIONS)" VOLUSPA=voluspa-covered ./test.sh; \ + fi; + +regress: + cd regress && ./test.sh + +other: + cd other && ./test.sh + +custom: + if [ -x custom ]; then \ + cd custom && ./test.sh; \ + fi; + +regen_baseline: + cd regress && ./regen_baseline.sh + cd other && ./regen_baseline.sh + if [ -x custom ]; then \ + cd custom && ./regen_baseline.sh; \ + fi; + +.PHONY: regress other custom +.NOTPARALLEL: diff --git a/tools/voluspa/tests/README.md b/tools/voluspa/tests/README.md new file mode 100644 index 00000000000..98c5420ac46 --- /dev/null +++ b/tools/voluspa/tests/README.md @@ -0,0 +1,16 @@ +# Tests + +This directory contains a series of sanity tests. These are simple tests comparing the output of a known good build/configuration to the latest locally built version of **voluspa**. + +The script "test.sh" will use the current build of the tool to regenerate the remap configs for a given yaml file; and do a diff between base/ and the new output. Generally, there should be no changes between runs. If there are expected changes, the baselines would need to be regenerated. The configurations tested range from very simple to something exercising all current configuration options. + +## Current Tests + +* simple/ + * Basic functionality +* all/ + * Every current configuration option exercising combinations of config types (normal and explicit) +* order/ + * Covers both ordering of properties by QPS and grouping via "role" +* parentchild/ + * A baseline of current parent/child config generation diff --git a/tools/voluspa/tests/other/all_bad/.gitignore b/tools/voluspa/tests/other/all_bad/.gitignore new file mode 100644 index 00000000000..9daeafb9864 --- /dev/null +++ b/tools/voluspa/tests/other/all_bad/.gitignore @@ -0,0 +1 @@ +test diff --git a/tools/voluspa/tests/other/all_bad/base/stderr.txt b/tools/voluspa/tests/other/all_bad/base/stderr.txt new file mode 100644 index 00000000000..dd3c698ba51 --- /dev/null +++ b/tools/voluspa/tests/other/all_bad/base/stderr.txt @@ -0,0 +1,14 @@ +voluspa: errors validating configs: + - alias http://(.*.myalias.com not unique (all_bad/simple2.conf) + - alias http://inbound.testing.example.com not unique (all_bad/simple2.conf) + - alias https://inbound.testing.example.com not unique (all_bad/simple2.conf) + - invalid regex URL: error parsing regexp: missing closing ): `http://(.*.myalias.com` (all_bad/simple2.conf) + - invalid regex: error parsing regexp: unexpected ): `# If Cookie : ADCDownloadAuth is missing then +# 302 to developer.example.com/unauthorized/ +cond %{COOKIE:Tasty} ="" [AND] +cond %{CLIENT-IP} ^127\.0\.0\.1|10\..*)/ [NOT] + set-redirect 302 https://inbound.example.com/unauthorized/ [L] +# If Origin serves a 000, 500 501 502 503 then +# 302 to inboud.example.com/unauthorized/` (all_bad/simple2.conf) + - property All not unique (all_bad/simple.conf) + - property All not unique (all_bad/simple2.conf) diff --git a/tools/voluspa/tests/other/all_bad/duplicated_property_name.conf b/tools/voluspa/tests/other/all_bad/duplicated_property_name.conf new file mode 100644 index 00000000000..3536d7c9fb9 --- /dev/null +++ b/tools/voluspa/tests/other/all_bad/duplicated_property_name.conf @@ -0,0 +1,13 @@ +schema_version: 1 +owner: nobody +rcpt: All +schemes: [https, http] + +mappings: + - alias: + - dupe.testing.example.com + origin: https://dupe.example.com + +rules: + default: + deny_methods: [ POST, PUT, DELETE ] diff --git a/tools/voluspa/tests/other/all_bad/simple.conf b/tools/voluspa/tests/other/all_bad/simple.conf new file mode 100644 index 00000000000..9da5d2e60af --- /dev/null +++ b/tools/voluspa/tests/other/all_bad/simple.conf @@ -0,0 +1,13 @@ +schema_version: '1.0' +owner: nobody +rcpt: All +schemes: [https, http] + +mappings: + - alias: + - inbound.testing.example.com + origin: https://origin.example.com + +rules: + default: + deny_methods: [ POST, PUT, DELETE ] diff --git a/tools/voluspa/tests/other/all_bad/simple2.conf b/tools/voluspa/tests/other/all_bad/simple2.conf new file mode 100644 index 00000000000..1fe7f8150b1 --- /dev/null +++ b/tools/voluspa/tests/other/all_bad/simple2.conf @@ -0,0 +1,25 @@ +schema_version: 1 +owner: nobody +rcpt: All +schemes: [https, http] + +mappings: + - alias: + - inbound.testing.example.com + origin: https://origin.example.com + - alias: + - http://(.*.myalias.com + origin: http://origin3.example.com + regex_map: true + +rules: + default: + deny_methods: [ POST, PUT, DELETE ] + header_rewrite: |- + # If Cookie : ADCDownloadAuth is missing then + # 302 to developer.example.com/unauthorized/ + cond %{COOKIE:Tasty} ="" [AND] + cond %{CLIENT-IP} ^127\.0\.0\.1|10\..*)/ [NOT] + set-redirect 302 https://inbound.example.com/unauthorized/ [L] + # If Origin serves a 000, 500 501 502 503 then + # 302 to inboud.example.com/unauthorized/ diff --git a/tools/voluspa/tests/other/cachekey_bad/base/stderr.txt b/tools/voluspa/tests/other/cachekey_bad/base/stderr.txt new file mode 100644 index 00000000000..9a5cc8b30cc --- /dev/null +++ b/tools/voluspa/tests/other/cachekey_bad/base/stderr.txt @@ -0,0 +1,3 @@ +voluspa: errors validating configs: + - invalid regex: error parsing regexp: missing closing ): `BYT\s([^,]+.*` (cachekey_bad/test.conf) + - invalid regex: unexpected / (cachekey_bad/test.conf) diff --git a/tools/voluspa/tests/other/cachekey_bad/test.conf b/tools/voluspa/tests/other/cachekey_bad/test.conf new file mode 100644 index 00000000000..3f9e41961c2 --- /dev/null +++ b/tools/voluspa/tests/other/cachekey_bad/test.conf @@ -0,0 +1,28 @@ +schema_version: 1 +owner: "" +reference: your reference notes here +rcpt: All +ssl_cert_names: + - /secrets/mysecret.txt + +mappings: + - origin: origin7.example.com + alias: + - myalias7.example.com + - myalias7.example.com/ran + - myalias7.example.com/bob + - /path/test.txt + +rules: + default: + cachekey: + include_params: ["param1", "param2"] + exclude_params: ["param1", "param2"] + include_headers: ["header1", "header2"] + include_cookies: ["cookie1", "cookie2"] + sort_query: true + remove_all_params: true + static_prefix: "SomeString" + regex_replace_path: '/.*/probe-info-ok.html/' + regex_replace_path_uri: '/http[s]?:\//[^\/]*.*\.html|.*).*/$1/' + capture_header: [ '/Authorization:AWS\s([^:]+).*/clientID:$1//', '/Authorization:BYT\s([^,]+.*/clientID:$1/' ] diff --git a/tools/voluspa/tests/other/duped/base/stderr.txt b/tools/voluspa/tests/other/duped/base/stderr.txt new file mode 100644 index 00000000000..fa0270f952c --- /dev/null +++ b/tools/voluspa/tests/other/duped/base/stderr.txt @@ -0,0 +1,2 @@ +voluspa: errors validating configs: + - alias http://inbound.example.com not unique for rule role "roles_edge_child" (duped/test.conf) diff --git a/tools/voluspa/tests/other/duped/test.conf b/tools/voluspa/tests/other/duped/test.conf new file mode 100644 index 00000000000..b06e3c1418d --- /dev/null +++ b/tools/voluspa/tests/other/duped/test.conf @@ -0,0 +1,27 @@ +schema_version: 1 +owner: owner +rcpt: streamer +parent_child: true + +rules: + default_child: + priority: streamingvideo + + default_child2: + priority: background + +explicit: + - alias: + - http://inbound.example.com + origin: http://origin.example.com + rule: default_child + role: roles_edge_child + + # NOTE: Duping above on purpose to trigger duplicate alias + - alias: + - http://inbound.example.com + origin: http://origin2.example.com + rule: default_child2 + role: roles_edge_child + +# vim: set et sw=2 ts=2 ft=yaml : diff --git a/tools/voluspa/tests/other/properties.sh b/tools/voluspa/tests/other/properties.sh new file mode 100644 index 00000000000..d903ee41f53 --- /dev/null +++ b/tools/voluspa/tests/other/properties.sh @@ -0,0 +1,19 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +CUST=(all_bad cachekey_bad duped unicode scheme) diff --git a/tools/voluspa/tests/other/regen_baseline.sh b/tools/voluspa/tests/other/regen_baseline.sh new file mode 100755 index 00000000000..96880d7fe4e --- /dev/null +++ b/tools/voluspa/tests/other/regen_baseline.sh @@ -0,0 +1,37 @@ +#!/bin/bash +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +. properties.sh + +if [ $# -gt 0 ]; then + CUST=($1) +fi + +CMD=../../voluspa +SCHEMA=../.. + +for i in ${CUST[@]}; do + DEST=./${i}/base + rm -rf $DEST + mkdir $DEST + ./$CMD --schema-location $SCHEMA --dest $DEST ${i}/*.conf 2> $DEST/stderr.txt +done + +# Ignore failures from voluspa +exit 0 diff --git a/tools/voluspa/tests/other/scheme/base/stderr.txt b/tools/voluspa/tests/other/scheme/base/stderr.txt new file mode 100644 index 00000000000..7195ca43c90 --- /dev/null +++ b/tools/voluspa/tests/other/scheme/base/stderr.txt @@ -0,0 +1,2 @@ +problem loading "scheme/test.conf": + - mappings.0.schemes.0: mappings.0.schemes.0 must be one of the following: "http", "https" diff --git a/tools/voluspa/tests/other/scheme/test.conf b/tools/voluspa/tests/other/scheme/test.conf new file mode 100644 index 00000000000..779a72a7174 --- /dev/null +++ b/tools/voluspa/tests/other/scheme/test.conf @@ -0,0 +1,9 @@ +schema_version: 1 +rcpt: seed +owner: "nobody" + +mappings: + - alias: + - inbound.example.com + origin: https://origin.example.com + schemes: [httpZs, http] diff --git a/tools/voluspa/tests/other/test.sh b/tools/voluspa/tests/other/test.sh new file mode 100755 index 00000000000..eca4975f2cf --- /dev/null +++ b/tools/voluspa/tests/other/test.sh @@ -0,0 +1,47 @@ +#!/bin/bash +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +. properties.sh + +if [ $# -gt 0 ]; then + CUST=($1) +fi + +# this expects 'make voluspa' to be run (make sanity_tests or make test does this) +if [ -z "$VOLUSPA" ]; then + VOLUSPA="voluspa" +fi + +CMD=../../$DEST/$VOLUSPA +SCHEMA=../.. + +for i in ${CUST[@]}; do + DEST=./${i}/test + rm -rf $DEST + mkdir $DEST + ./$CMD --schema-location $SCHEMA --dest $DEST ${i}/*.conf 2> $DEST/stderr.txt + if [ "$VOLUSPA" == "voluspa" ]; then + diff -r ${i}/base $DEST + if [ $? != 0 ]; then + exit 1 + fi + else + mv system.out profile_${i}.out + fi +done diff --git a/tools/voluspa/tests/other/unicode/base/stderr.txt b/tools/voluspa/tests/other/unicode/base/stderr.txt new file mode 100644 index 00000000000..0327c36679d --- /dev/null +++ b/tools/voluspa/tests/other/unicode/base/stderr.txt @@ -0,0 +1 @@ +problem loading "unicode/httpbin.conf": Could not load YAML file. err=non-ASCII characters not supported: invalid rune "🍎" diff --git a/tools/voluspa/tests/other/unicode/httpbin.conf b/tools/voluspa/tests/other/unicode/httpbin.conf new file mode 100644 index 00000000000..c630cc59e0e --- /dev/null +++ b/tools/voluspa/tests/other/unicode/httpbin.conf @@ -0,0 +1,16 @@ +schema_version: 1 +owner: Property Owner +reference: For testing ATS behavior 🍎 +rcpt: httpbin +schemes: [http, https] + +mappings: + - alias: + - httpbin.example.com + origin: https://httpbin.edge.example.com + +rules: + default: + priority: background + origin_host_header: alias + deny_methods: [ CONNECT ] diff --git a/tools/voluspa/tests/regress/all/base/all/all.config b/tools/voluspa/tests/regress/all/base/all/all.config new file mode 100644 index 00000000000..fea885654ab --- /dev/null +++ b/tools/voluspa/tests/regress/all/base/all/all.config @@ -0,0 +1,196 @@ +# Code generated by Voluspa. DO NOT EDIT. + +map http://myalias7.example.com/bob \ + http://origin7.example.com + +map http://myalias7.example.com/ran \ + http://origin7.example.com + +map http://myalias7.example.com \ + http://origin7.example.com + +map /path/test.txt \ + http://origin7.example.com + +map http://redirect.example.com \ + http://origin7.example.com \ + @plugin=regex_remap.so @pparam=all/redirect9.config + +map https://myalias1.example.com \ + https://origin2.example.com + +map https://myalias2.example.com \ + https://origin2.example.com + +regex_map http://(.*).example.com \ + http://origin3.example.com \ + @plugin=access_control.so @pparam=--symmetric-keys-map=hmac_keys.txt \ + @pparam=--check-cookie=TokenCookie \ + @pparam=--extract-subject-to-header=@TokenSubject \ + @pparam=--extract-tokenid-to-header=@TokenId \ + @pparam=--extract-status-to-header=@TokenStatus \ + @pparam=--token-response-header=TokenRespHdr \ +{% if salt.pillar.get("roles_trafficserver_cache_promote") %} @plugin=cache_promote.so @pparam=--policy=lru @pparam=--hits=5 @pparam=--buckets=100000 \{% else %} \{% endif %} + @plugin=cachekey.so @pparam=--include-params=param1,param2 \ + @pparam=--include-cookies=cookie1,cookie2 \ + @pparam=--include-headers=header1,header2 \ + @pparam=--exclude-params=param1,param2 \ + @pparam=--static-prefix=SomeString \ + @pparam=--sort-params=true \ + @pparam=--remove-all-params=true \ + @pparam=--capture-path=/.*/probe-info-ok.html/ \ + @pparam=--capture-path-uri=/http[s]?:\/\/[^\/]*(.*\.html|.*).*/$1/ \ + @pparam=--capture-header=Authorization:/AWS\s([^:]+).*/clientID:$1/ \ + @pparam=--capture-header=Authorization:/BYT\s([^,]+).*/clientID:$1/ \ + @plugin=gzip.so @pparam=all/gzip2.config \ + @plugin=escalate.so @pparam=--pristine \ + @pparam=401,403,404,407,410,500,501,502,503,504,505:assets-origin.subdomain.example.com \ + @plugin=regex_remap.so @pparam=all/strip_query2.config \ + @plugin=volcano.so @pparam=--fetch-count=10 @pparam=--fetch-max=1000 @pparam=--api-header=header1 @pparam=--replace-host=host1 @pparam=--name-space=ns1 @pparam=--metrics-prefix=prefix1 @pparam=--exact-match=true @pparam=--log-name=log1 {% if salt.pillar.get("roles_video_fetch_backend") %} @pparam=--fetch-policy=simple @pparam=--front=false \{% else %} \{% endif %} {% if salt.pillar.get("roles_video_fetch_frontend") %} @pparam=--fetch-policy=lru:123 @pparam=--fetch-path-pattern=/(.*-)(\d+)(.*)/$1{$2+2}$3/ @pparam=--fetch-path-pattern=/(.*)(\d{3})(_.*\.jpg.*)/$1{$2+1:3}$3/ @pparam=--front=true \{% else %} \{% endif %} + @plugin=regex_remap.so @pparam=all/regex2.config \ + @plugin=s3_auth.so @pparam=--config @pparam=certs/s3.config \ + @pparam=--virtual_host \ + @pparam=--version=4 \ + @pparam=--v4-include-headers=header1,header2 \ + @pparam=--v4-exclude-headers=header3,header4 \ + @pparam=--v4-region-map=region_map_test.config \ + @plugin=conf_remap.so @pparam=proxy.config.http.cache.generation=3 \ + @pparam=proxy.config.http.connect_attempts_timeout=600 \ + @pparam=proxy.config.http.transaction_no_activity_timeout_in=600 \ + @pparam=proxy.config.http.transaction_no_activity_timeout_out=600 \ + @pparam=proxy.config.http.cache.http=0 \ + @pparam=proxy.config.http.negative_caching_lifetime=3600 \ + @pparam=proxy.config.url_remap.pristine_host_hdr=0 \ + @pparam=proxy.config.http.connect_attempts_timeout=600 \ + @pparam=proxy.config.http.transaction_no_activity_timeout_in=600 \ + @pparam=proxy.config.http.transaction_no_activity_timeout_out=600 \ + @plugin=header_rewrite.so @pparam=all/hdrs2.config \ + @src_ip=10.153.0.0-10.153.255.255 @src_ip=10.179.0.0-10.179.255.255 @action=allow \ + @action=allow @method=PURGE \ + @action=deny @method=POST @method=PUT @method=DELETE + +map http://myalias4.example.com \ + http://origin4.example.com \ + @plugin=access_control.so @pparam=--symmetric-keys-map=hmac_keys.txt \ + @pparam=--check-cookie=TokenCookie \ + @pparam=--extract-subject-to-header=@TokenSubject \ + @pparam=--extract-tokenid-to-header=@TokenId \ + @pparam=--extract-status-to-header=@TokenStatus \ + @pparam=--token-response-header=TokenRespHdr \ +{% if salt.pillar.get("roles_trafficserver_cache_promote") %} @plugin=cache_promote.so @pparam=--policy=lru @pparam=--hits=5 @pparam=--buckets=100000 \{% else %} \{% endif %} + @plugin=cachekey.so @pparam=--include-params=param1,param2 \ + @pparam=--include-cookies=cookie1,cookie2 \ + @pparam=--include-headers=header1,header2 \ + @pparam=--exclude-params=param1,param2 \ + @pparam=--static-prefix=SomeString \ + @pparam=--sort-params=true \ + @pparam=--remove-all-params=true \ + @pparam=--capture-path=/.*/probe-info-ok.html/ \ + @pparam=--capture-path-uri=/http[s]?:\/\/[^\/]*(.*\.html|.*).*/$1/ \ + @pparam=--capture-header=Authorization:/AWS\s([^:]+).*/clientID:$1/ \ + @pparam=--capture-header=Authorization:/BYT\s([^,]+).*/clientID:$1/ \ + @plugin=gzip.so @pparam=all/gzip2.config \ + @plugin=escalate.so @pparam=--pristine \ + @pparam=401,403,404,407,410,500,501,502,503,504,505:assets-origin.subdomain.example.com \ + @plugin=regex_remap.so @pparam=all/strip_query2.config \ + @plugin=volcano.so @pparam=--fetch-count=10 @pparam=--fetch-max=1000 @pparam=--api-header=header1 @pparam=--replace-host=host1 @pparam=--name-space=ns1 @pparam=--metrics-prefix=prefix1 @pparam=--exact-match=true @pparam=--log-name=log1 {% if salt.pillar.get("roles_video_fetch_backend") %} @pparam=--fetch-policy=simple @pparam=--front=false \{% else %} \{% endif %} {% if salt.pillar.get("roles_video_fetch_frontend") %} @pparam=--fetch-policy=lru:123 @pparam=--fetch-path-pattern=/(.*-)(\d+)(.*)/$1{$2+2}$3/ @pparam=--fetch-path-pattern=/(.*)(\d{3})(_.*\.jpg.*)/$1{$2+1:3}$3/ @pparam=--front=true \{% else %} \{% endif %} + @plugin=regex_remap.so @pparam=all/regex2.config \ + @plugin=s3_auth.so @pparam=--config @pparam=certs/s3.config \ + @pparam=--virtual_host \ + @pparam=--version=4 \ + @pparam=--v4-include-headers=header1,header2 \ + @pparam=--v4-exclude-headers=header3,header4 \ + @pparam=--v4-region-map=region_map_test.config \ + @plugin=conf_remap.so @pparam=proxy.config.http.cache.generation=3 \ + @pparam=proxy.config.http.connect_attempts_timeout=600 \ + @pparam=proxy.config.http.transaction_no_activity_timeout_in=600 \ + @pparam=proxy.config.http.transaction_no_activity_timeout_out=600 \ + @pparam=proxy.config.http.cache.http=0 \ + @pparam=proxy.config.http.negative_caching_lifetime=3600 \ + @pparam=proxy.config.url_remap.pristine_host_hdr=0 \ + @pparam=proxy.config.http.connect_attempts_timeout=600 \ + @pparam=proxy.config.http.transaction_no_activity_timeout_in=600 \ + @pparam=proxy.config.http.transaction_no_activity_timeout_out=600 \ + @plugin=header_rewrite.so @pparam=all/hdrs2.config \ + @src_ip=10.153.0.0-10.153.255.255 @src_ip=10.179.0.0-10.179.255.255 @action=allow \ + @action=allow @method=PURGE \ + @action=deny @method=POST @method=PUT @method=DELETE + +map https://myalias4.example.com \ + https://origin4.example.com \ + @plugin=access_control.so @pparam=--symmetric-keys-map=hmac_keys.txt \ + @pparam=--check-cookie=TokenCookie \ + @pparam=--extract-subject-to-header=@TokenSubject \ + @pparam=--extract-tokenid-to-header=@TokenId \ + @pparam=--extract-status-to-header=@TokenStatus \ + @pparam=--token-response-header=TokenRespHdr \ +{% if salt.pillar.get("roles_trafficserver_cache_promote") %} @plugin=cache_promote.so @pparam=--policy=lru @pparam=--hits=5 @pparam=--buckets=100000 \{% else %} \{% endif %} + @plugin=cachekey.so @pparam=--include-params=param1,param2 \ + @pparam=--include-cookies=cookie1,cookie2 \ + @pparam=--include-headers=header1,header2 \ + @pparam=--exclude-params=param1,param2 \ + @pparam=--static-prefix=SomeString \ + @pparam=--sort-params=true \ + @pparam=--remove-all-params=true \ + @pparam=--capture-path=/.*/probe-info-ok.html/ \ + @pparam=--capture-path-uri=/http[s]?:\/\/[^\/]*(.*\.html|.*).*/$1/ \ + @pparam=--capture-header=Authorization:/AWS\s([^:]+).*/clientID:$1/ \ + @pparam=--capture-header=Authorization:/BYT\s([^,]+).*/clientID:$1/ \ + @plugin=gzip.so @pparam=all/gzip2.config \ + @plugin=escalate.so @pparam=--pristine \ + @pparam=401,403,404,407,410,500,501,502,503,504,505:assets-origin.subdomain.example.com \ + @plugin=regex_remap.so @pparam=all/strip_query2.config \ + @plugin=volcano.so @pparam=--fetch-count=10 @pparam=--fetch-max=1000 @pparam=--api-header=header1 @pparam=--replace-host=host1 @pparam=--name-space=ns1 @pparam=--metrics-prefix=prefix1 @pparam=--exact-match=true @pparam=--log-name=log1 {% if salt.pillar.get("roles_video_fetch_backend") %} @pparam=--fetch-policy=simple @pparam=--front=false \{% else %} \{% endif %} {% if salt.pillar.get("roles_video_fetch_frontend") %} @pparam=--fetch-policy=lru:123 @pparam=--fetch-path-pattern=/(.*-)(\d+)(.*)/$1{$2+2}$3/ @pparam=--fetch-path-pattern=/(.*)(\d{3})(_.*\.jpg.*)/$1{$2+1:3}$3/ @pparam=--front=true \{% else %} \{% endif %} + @plugin=regex_remap.so @pparam=all/regex2.config \ + @plugin=s3_auth.so @pparam=--config @pparam=certs/s3.config \ + @pparam=--virtual_host \ + @pparam=--version=4 \ + @pparam=--v4-include-headers=header1,header2 \ + @pparam=--v4-exclude-headers=header3,header4 \ + @pparam=--v4-region-map=region_map_test.config \ + @plugin=conf_remap.so @pparam=proxy.config.http.cache.generation=3 \ + @pparam=proxy.config.http.connect_attempts_timeout=600 \ + @pparam=proxy.config.http.transaction_no_activity_timeout_in=600 \ + @pparam=proxy.config.http.transaction_no_activity_timeout_out=600 \ + @pparam=proxy.config.http.cache.http=0 \ + @pparam=proxy.config.http.negative_caching_lifetime=3600 \ + @pparam=proxy.config.url_remap.pristine_host_hdr=0 \ + @pparam=proxy.config.http.connect_attempts_timeout=600 \ + @pparam=proxy.config.http.transaction_no_activity_timeout_in=600 \ + @pparam=proxy.config.http.transaction_no_activity_timeout_out=600 \ + @plugin=header_rewrite.so @pparam=all/hdrs2.config \ + @src_ip=10.153.0.0-10.153.255.255 @src_ip=10.179.0.0-10.179.255.255 @action=allow \ + @action=allow @method=PURGE \ + @action=deny @method=POST @method=PUT @method=DELETE + +map http://myalias5.example.com \ + http://origin4.example.com \ + @plugin=header_rewrite.so @pparam=all/hdrs6.config + +{% if salt.pillar.get("roles_edge_child") %} + +map http://myalias6.example.com \ + http://origin6.example.com \ + @plugin=volcano.so {% if salt.pillar.get("roles_video_fetch_backend") %} @pparam=--fetch-policy=simple @pparam=--front=false \{% else %} \{% endif %} {% if salt.pillar.get("roles_video_fetch_frontend") %} @pparam=--fetch-policy=lru:123 @pparam=--front=true \{% else %} \{% endif %} + +{% endif %} + +map http://myalias8.example.com \ + http://origin8.example.com \ + @plugin=header_rewrite.so @pparam=all/hdrs10.config + +map http://myalias9.example.com \ + http://origin9.example.com \ + @plugin=s3_auth.so @pparam=--config @pparam=certs/s3.config \ + @pparam=--virtual_host \ + @pparam=--version=4 \ + @pparam=--v4-region-map=region_map_test.config \ + @pparam=--v4-exclude-headers=x-forwarded-for,forwarded,via,authorization + +map http://myaliasa.example.com \ + http://origina.example.com \ + @plugin=s3_auth.so @pparam=--config @pparam=certs/s3.config \ + @pparam=--virtual_host \ + @pparam=--version=2 \ + @pparam=--v4-region-map=region_map_test.config + diff --git a/tools/voluspa/tests/regress/all/base/all/gzip2.config b/tools/voluspa/tests/regress/all/base/all/gzip2.config new file mode 100644 index 00000000000..6dc3e065479 --- /dev/null +++ b/tools/voluspa/tests/regress/all/base/all/gzip2.config @@ -0,0 +1,10 @@ +# Code generated by Voluspa. DO NOT EDIT. + +enabled true +cache false +remove-accept-encoding true +flush true +minimum-content-length 532 +supported-algorithms br,gzip +compressible-content-type *javascript* +compressible-content-type text/* diff --git a/tools/voluspa/tests/regress/all/base/all/hdrs10.config b/tools/voluspa/tests/regress/all/base/all/hdrs10.config new file mode 100644 index 00000000000..7ae70a9452b --- /dev/null +++ b/tools/voluspa/tests/regress/all/base/all/hdrs10.config @@ -0,0 +1,4 @@ +# Code generated by Voluspa. DO NOT EDIT. + +cond %{REMAP_PSEUDO_HOOK} + set-header @cdnlog "private" diff --git a/tools/voluspa/tests/regress/all/base/all/hdrs2.config b/tools/voluspa/tests/regress/all/base/all/hdrs2.config new file mode 100644 index 00000000000..1f5c961bd93 --- /dev/null +++ b/tools/voluspa/tests/regress/all/base/all/hdrs2.config @@ -0,0 +1,52 @@ +# Code generated by Voluspa. DO NOT EDIT. + +cond %{READ_RESPONSE_HDR_HOOK} + add-header X-Client-Protocol % +cond %{SEND_REQUEST_HDR_HOOK} + add-header X-TestOriginHeader "1" +cond %{SEND_RESPONSE_HDR_HOOK} [AND] + cond %{PATH} /.*\.crl$/ + set-header Content-Type application/pkix-crl +cond %{SEND_RESPONSE_HDR_HOOK} [AND] + cond %{PATH} /.*\.css$/ + set-header Content-Type text/css +cond %{SEND_RESPONSE_HDR_HOOK} [AND] + cond %{PATH} /.*\.json$/ + set-header Content-Type application/json + +cond %{READ_RESPONSE_HDR_HOOK} [AND] +cond %{HEADER:Cache-Control} ="" [AND] +cond %{STATUS} >199 [AND] +cond %{STATUS} <400 + set-header Cache-Control "max-age=3600, public" +cond %{READ_RESPONSE_HDR_HOOK} [AND] +cond %{STATUS} >199 [AND] +cond %{STATUS} <400 + set-header Cache-Control "max-age=0, public" +cond %{SEND_REQUEST_HDR_HOOK} + add-header Authorization "Basic 123412341234" [L] + +cond %{REMAP_PSEUDO_HOOK} + set-header Bazinga1 %{COOKIE:NameOfCookie1} + set-header Bazinga2 %{COOKIE:NameOfCookie2} +cond %{REMAP_PSEUDO_HOOK} + set-header Bazinga3 %{HEADER:NameOfHeader1} + set-header Bazinga4 %{HEADER:NameOfHeader2} + +cond %{REMAP_PSEUDO_HOOK} + set-conn-dscp 8 +cond %{READ_RESPONSE_HDR_HOOK} + set-header @Original-Cache-Control %{HEADER:Cache-Control} + set-header Cache-Control %{HEADER:MyProxyHeader} + +cond %{SEND_RESPONSE_HDR_HOOK} + set-header Cache-Control %{HEADER:@Original-Cache-Control} + rm-header Proxy-Cache-Control +cond %{REMAP_PSEUDO_HOOK} + rm-header Proxy +cond %{SEND_REQUEST_HDR_HOOK} + rm-header Proxy +cond %{READ_RESPONSE_HDR_HOOK} + set-header X-Client-Protocol % +cond %{SEND_REQUEST_HDR_HOOK} + set-header X-TestOriginHeader "1" diff --git a/tools/voluspa/tests/regress/all/base/all/hdrs6.config b/tools/voluspa/tests/regress/all/base/all/hdrs6.config new file mode 100644 index 00000000000..d1606fafb7e --- /dev/null +++ b/tools/voluspa/tests/regress/all/base/all/hdrs6.config @@ -0,0 +1,4 @@ +# Code generated by Voluspa. DO NOT EDIT. + +cond %{REMAP_PSEUDO_HOOK} + set-header @ReceiptService "override{{hosttype}}" diff --git a/tools/voluspa/tests/regress/all/base/all/redirect9.config b/tools/voluspa/tests/regress/all/base/all/redirect9.config new file mode 100644 index 00000000000..866f0304a09 --- /dev/null +++ b/tools/voluspa/tests/regress/all/base/all/redirect9.config @@ -0,0 +1,3 @@ +# Code generated by Voluspa. DO NOT EDIT. + +(.*) https://example.com/filenotfound @status=302 diff --git a/tools/voluspa/tests/regress/all/base/all/regex2.config b/tools/voluspa/tests/regress/all/base/all/regex2.config new file mode 100644 index 00000000000..9889c40e6ce --- /dev/null +++ b/tools/voluspa/tests/regress/all/base/all/regex2.config @@ -0,0 +1,3 @@ +# Code generated by Voluspa. DO NOT EDIT. + +^/([^/]+)/(.*) https://$1.s3.amazonaws.com/$2 diff --git a/tools/voluspa/tests/regress/all/base/all/strip_query2.config b/tools/voluspa/tests/regress/all/base/all/strip_query2.config new file mode 100644 index 00000000000..7b0312493d4 --- /dev/null +++ b/tools/voluspa/tests/regress/all/base/all/strip_query2.config @@ -0,0 +1,3 @@ +# Code generated by Voluspa. DO NOT EDIT. + +. $s://$t/$P diff --git a/tools/voluspa/tests/regress/all/base/hosting.config_default b/tools/voluspa/tests/regress/all/base/hosting.config_default new file mode 100644 index 00000000000..4dd9b032a1c --- /dev/null +++ b/tools/voluspa/tests/regress/all/base/hosting.config_default @@ -0,0 +1,7 @@ +# Code generated by Voluspa. DO NOT EDIT. + +{% if salt.pillar.get("roles_trafficserver_ramdisk") %} +hostname=* {{ default_volumes }} +{% else %} +# hosting.config disabled on this host +{% endif %} diff --git a/tools/voluspa/tests/regress/all/base/ssl_multicert.config_default b/tools/voluspa/tests/regress/all/base/ssl_multicert.config_default new file mode 100644 index 00000000000..4fad5702530 --- /dev/null +++ b/tools/voluspa/tests/regress/all/base/ssl_multicert.config_default @@ -0,0 +1,9 @@ +# Code generated by Voluspa. DO NOT EDIT. + +{% if not salt.pillar.get('roles_uat') %} + +# your reference notes here +ssl_cert_name=/secrets/mysecret.txt + +{% endif %} + diff --git a/tools/voluspa/tests/regress/all/test.conf b/tools/voluspa/tests/regress/all/test.conf new file mode 100644 index 00000000000..71461394018 --- /dev/null +++ b/tools/voluspa/tests/regress/all/test.conf @@ -0,0 +1,239 @@ +schema_version: 1 +owner: "" +reference: your reference notes here +rcpt: All +ssl_cert_names: + - /secrets/mysecret.txt + +rules: + default: + add_header: "X-Client-Protocol %" + add_header_origin: X-TestOriginHeader "1" + set_header: "X-Client-Protocol %" + set_header_origin: X-TestOriginHeader "1" + remove_header: "Proxy" + remove_header_origin: "Proxy" + allow_ip: [10.153.0.0-10.153.255.255, 10.179.0.0-10.179.255.255] + allow_ip_parent: [10.253.0.0-10.253.255.255, 10.254.0.0-10.254.255.255] + strip_query: true + force_ttl: 0s + proxy_cache_control: "MyProxyHeader" + origin_host_header: origin + disable_cache: true + deny_methods: [ POST, PUT, DELETE ] + allow_methods: [ PURGE ] + regex_remap: "^/([^/]+)/(.*) https://$1.s3.amazonaws.com/$2" + s3: + path: certs/s3.config + virtual_host: true + version: 4 + v4_include_headers: ["header1", "header2"] + v4_exclude_headers: ["header3", "header4"] + region_map: region_map_test.config + video_background_fetch: + fetch_count: 10 + fetch_max: 1000 + api_header: header1 + replace_host: host1 + name_space: ns1 + metrics_prefix: prefix1 + exact_match: true + log_name: log1 + frontend: + fetch_path_pattern: + - '/(.*-)(\d+)(.*)/$1{$2+2}$3/' + - /(.*)(\d{3})(_.*\.jpg.*)/$1{$2+1:3}$3/ + fetch_policy: lru:123 + backend: + fetch_policy: simple + + cache_promote: + policy: lru + lru_hits: 5 + lru_buckets: 100000 + + cachekey: + include_params: ["param1", "param2"] + exclude_params: ["param1", "param2"] + include_headers: ["header1", "header2"] + include_cookies: ["cookie1", "cookie2"] + sort_query: true + remove_all_params: true + static_prefix: "SomeString" + regex_replace_path: '/.*/probe-info-ok.html/' + regex_replace_path_uri: '/http[s]?:\/\/[^\/]*(.*\.html|.*).*/$1/' + capture_header: [ 'Authorization:/AWS\s([^:]+).*/clientID:$1/', 'Authorization:/BYT\s([^,]+).*/clientID:$1/' ] + failover: + domain: assets-origin.subdomain.example.com + status_codes: [401, 403, 404, 407, 410, 500, 501, 502, 503, 504, 505] + host_header: origin + priority: background + default_ttl: 60m + content_type_forge: + '.*\.json$': application/json + '.*\.css$': text/css + '.*\.crl$': application/pkix-crl + conf_remap: "CONFIG proxy.config.http.connect_attempts_timeout INT 600\n + CONFIG proxy.config.http.transaction_no_activity_timeout_in INT 600\n + CONFIG proxy.config.http.transaction_no_activity_timeout_out INT 600" + header_rewrite: "cond %{SEND_REQUEST_HDR_HOOK}\n + add-header Authorization \"Basic 123412341234\" [L]" + log_cookie: + Logger-cookie-Name1: NameOfCookie1 + Logger-cookie-Name2: NameOfCookie2 + log_header: + Logger-header-Name1: NameOfHeader1 + Logger-header-Name2: NameOfHeader2 + negative_ttl: 60m + transaction_timeout: 10m + edge_compression: + enabled: true + flush: true + remove_accept_encoding: true + minimum_content_length: 532 + algorithms: [gzip, br] + static_origin_compressible_content_type: + - "text/*" + - "*javascript*" + access_approval: + raw: |- + @pparam=--symmetric-keys-map=hmac_keys.txt + @pparam=--check-cookie=TokenCookie + @pparam=--extract-subject-to-header=@TokenSubject + @pparam=--extract-tokenid-to-header=@TokenId + @pparam=--extract-status-to-header=@TokenStatus + @pparam=--token-response-header=TokenRespHdr + cache_version: 3 + + no_receipts: + receipts: + enabled: false + + override_receipt: + receipts: + name: override + + # Redirect takes one redirect url and status code. + # If status code is empty, it takes default value of 302. + redirect_this: + redirect: + url: https://example.com/filenotfound + + preserve_host_header: + origin_host_header: alias + receipts: + enabled: false + priority: foreground + s3: + path: /opt/ats/etc/trafficserver/certs/s3.config + virtual_host: false + + escalate_pristine: + failover: + domain: assets-origin.subdomain.example.com + status_codes: [401, 403, 404, 407, 410, 500, 501, 502, 503, 504, 505] + host_header: origin + + streamingaudio: + priority: streamingaudio + + streamingvideo: + priority: streamingvideo + + roles_within_roles: + video_background_fetch: + frontend: + fetch_policy: lru:123 + backend: + fetch_policy: simple + + cache_promote: + cache_promote: + policy: lru + lru_hits: 5 + lru_buckets: 100000 + + rlog_type: + log_type: private + + storage_volume_ram: + storage_volume: ramdisk_volume + + storage_volume_disk: + storage_volume: disk_volume + + raw_lua: + lua: |- + function do_remap() + local to_host = ts.remap.get_to_url_host() + ts.debug(to_host) + return 0 + end + + echo_cors_test: + echo_cors: "Origin" + + s3_v4: + s3: + path: certs/s3.config + virtual_host: true + version: 4 + region_map: region_map_test.config + + + s3_v2: + s3: + path: certs/s3.config + virtual_host: true + version: 2 + region_map: region_map_test.config + + +explicit: + - origin: https://origin2.example.com + alias: + - https://myalias1.example.com + - https://myalias2.example.com + rule: no_receipts + - origin: http://origin3.example.com + alias: + - http://(.*).example.com + regex_map: true + - origin: origin4.example.com + alias: + - myalias4.example.com + schemes: [http, https] + - origin: origin4.example.com + alias: + - myalias5.example.com + rule: override_receipt + - origin: origin6.example.com + alias: + - myalias6.example.com + role: roles_edge_child + rule: roles_within_roles + - origin: origin8.example.com + alias: + - myalias8.example.com + rule: rlog_type + - origin: origin9.example.com + alias: + - myalias9.example.com + rule: s3_v4 + - origin: origina.example.com + alias: + - myaliasa.example.com + rule: s3_v2 + +mappings: + - origin: origin7.example.com + alias: + - myalias7.example.com + - myalias7.example.com/ran + - myalias7.example.com/bob + - /path/test.txt + rule: no_receipts + - origin: origin7.example.com + alias: + - redirect.example.com + rule: redirect_this diff --git a/tools/voluspa/tests/regress/properties.sh b/tools/voluspa/tests/regress/properties.sh new file mode 100644 index 00000000000..f5c3cc28307 --- /dev/null +++ b/tools/voluspa/tests/regress/properties.sh @@ -0,0 +1,18 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +CUST=(all) diff --git a/tools/voluspa/tests/regress/regen_baseline.sh b/tools/voluspa/tests/regress/regen_baseline.sh new file mode 100755 index 00000000000..3733ceb352f --- /dev/null +++ b/tools/voluspa/tests/regress/regen_baseline.sh @@ -0,0 +1,36 @@ +#!/bin/bash +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +set +e + +. properties.sh + +if [ $# -gt 0 ]; then + CUST=($1) +fi + +CMD=../../voluspa +SCHEMA=../.. + +for i in ${CUST[@]}; do + DEST=./${i}/base + rm -rf $DEST + mkdir $DEST + ./$CMD --schema-location $SCHEMA --dest $DEST ${i}/*.conf +done diff --git a/tools/voluspa/tests/regress/test.sh b/tools/voluspa/tests/regress/test.sh new file mode 100755 index 00000000000..5e6b66890ea --- /dev/null +++ b/tools/voluspa/tests/regress/test.sh @@ -0,0 +1,52 @@ +#!/bin/bash +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +. properties.sh + +if [ $# -gt 0 ]; then + CUST=($1) +fi + +# this expects 'make voluspa' to be run (make sanity_tests or make test does this) +if [ -z "$VOLUSPA" ]; then + VOLUSPA="voluspa" +fi + +CMD=../../$DEST/$VOLUSPA +SCHEMA=../.. + +for i in ${CUST[@]}; do + DEST=./${i}/test + rm -rf $DEST + mkdir $DEST + ./$CMD --schema-location $SCHEMA --dest $DEST ${i}/*.conf > /dev/null + if [ "$VOLUSPA" == "voluspa" ]; then + if [ $? != 0 ]; then + exit 1 + fi + + diff -x .gitignore -x \*.swp -r ${i}/base $DEST + if [ $? != 0 ]; then + exit 1 + fi + else + mv system.out profile_${i}.out + fi +done + diff --git a/tools/voluspa/toplevel_remaps.go b/tools/voluspa/toplevel_remaps.go new file mode 100644 index 00000000000..102d5621662 --- /dev/null +++ b/tools/voluspa/toplevel_remaps.go @@ -0,0 +1,197 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package voluspa + +import ( + "bytes" + "fmt" + "sort" + "strings" +) + +type TopLevelRemapGenerator struct { + options *Options +} + +func newTopLevelRemapGenerator(options *Options) *TopLevelRemapGenerator { + return &TopLevelRemapGenerator{options: options} +} + +func (tlrg *TopLevelRemapGenerator) formatDescriptionComment(config *CustomerConfig) string { + var domain string + if strings.HasPrefix(config.Remaps[0].IncomingURL, "http") { + parts := strings.Split(config.Remaps[0].IncomingURL, "/") + if len(parts) >= 2 { + domain = parts[2] + } + } + + var buf bytes.Buffer + buf.WriteString("############################################################################\n") + buf.WriteString(fmt.Sprintf("# Host: %s\n", domain)) + buf.WriteString(fmt.Sprintf("# Owner: %s\n", config.owner)) + if len(config.reference) > 0 { + buf.WriteString(fmt.Sprintf("# Reference: %s\n", config.reference)) + } + buf.WriteString("############################################################################\n") + return buf.String() +} + +func (tlrg *TopLevelRemapGenerator) filenameForRole(role, cdn, parentOrChild string) string { + var buf bytes.Buffer + if cdn == DefaultCDN || cdn == NoCDN { + buf.WriteString("remap_") + } else { + buf.WriteString(fmt.Sprintf("%s_remap_", cdn)) + } + + if len(role) > 0 { + buf.WriteString(fmt.Sprintf("%s_", role)) + } + + buf.WriteString(fmt.Sprintf("%s_include.config", parentOrChild)) + + return buf.String() +} + +// groupedRemaps returns true if a Group has been explicitly set +func groupedRemaps(remaps []Remap) bool { + for _, remap := range remaps { + if remap.Group != DefaultGroup { + return true + } + } + return false +} + +func (tlrg *TopLevelRemapGenerator) Do(parsedConfigs []*CustomerConfig, merge bool) ([]ManagedFile, error) { + if merge { + return tlrg.get(parsedConfigs, NoRole, NoCDN) + } + + if len(parsedConfigs) == 0 { + return nil, ErrMinimumConfigsNotMet + } + + // Generating the include file is next to useless for 1 file + if len(parsedConfigs) == 1 { + return nil, nil + } + + // group properties by CDN and role and generate includeable files for each role/parent/child combo + grouped := make(map[string]map[string][]*CustomerConfig) + + for _, config := range parsedConfigs { + if config.lifecycle == Retired { + continue + } + + for cdn := range config.cdn { + + if _, exists := grouped[cdn]; !exists { + grouped[cdn] = make(map[string][]*CustomerConfig) + } + + role := config.role + grouped[cdn][role] = append(grouped[cdn][role], config) + } + } + var managedFiles []ManagedFile + for cdn, cdnGrouped := range grouped { + for role, configs := range cdnGrouped { + sort.Stable(sort.Reverse(byQPSAndName(configs))) + + remaps, err := tlrg.get(configs, role, cdn) + if err != nil { + return nil, err + } + managedFiles = append(managedFiles, remaps...) + } + } + + return managedFiles, nil + +} + +func (tlrg *TopLevelRemapGenerator) get(configs []*CustomerConfig, role, cdn string) ([]ManagedFile, error) { + var managedFiles []ManagedFile + + for _, parentOrChild := range []string{Child, Parent} { + + var buf bytes.Buffer + buf.WriteString("# START voluspa-generated config\n") + + if len(role) == 0 { + buf.WriteString("\n") + } else { + buf.WriteString(fmt.Sprintf("# ROLE: %s\n\n", role)) + } + + for _, parsedConfig := range configs { + if _, exists := parsedConfig.cdn[cdn]; !exists && cdn != NoCDN { + continue + } + + // skip grouped remap sets as the output cannot be used + // without knowledge beyond the scope of voluspa + // (feature used by cdn-icloud-content-uat) + if groupedRemaps(parsedConfig.Remaps) { + continue + } + + property := strings.ToLower(parsedConfig.property) + buf.WriteString(tlrg.formatDescriptionComment(parsedConfig)) + + filename := property + + if parentOrChild == Child || !parsedConfig.parentChild { + buf.WriteString(fmt.Sprintf(".include %s/%s.config\n", property, filename)) + } else { + buf.WriteString(fmt.Sprintf(".include %s/%s_parent.config\n", property, filename)) + } + buf.WriteString("\n") + } + buf.WriteString("# END voluspa-generated config\n\n") + + fileName := tlrg.filenameForRole(role, cdn, parentOrChild) + + var remoteFilename string + if len(role) > 0 { + remoteFilename = fmt.Sprintf("remap_voluspa_%s.config", role) + } else { + remoteFilename = "remap_voluspa.config" + } + + managedFiles = append(managedFiles, NewManagedFile(fileName, remoteFilename, cdn, role, "", &buf, tlrg.configType(parentOrChild))) + } + + return managedFiles, nil +} + +func (tlrg *TopLevelRemapGenerator) configType(val string) ConfigLocation { + switch val { + case Child: + return ChildConfig + case Parent: + return ParentConfig + default: + panic(fmt.Errorf("Unhandled configType '%s'", val)) + } +} diff --git a/tools/voluspa/util.go b/tools/voluspa/util.go new file mode 100644 index 00000000000..4d5e984c04f --- /dev/null +++ b/tools/voluspa/util.go @@ -0,0 +1,32 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package voluspa + +// groupCustomerConfigsByCDN groups CustomerConfigs by CDN +func groupCustomerConfigsByCDN(parsedConfigs []*CustomerConfig) map[string][]*CustomerConfig { + // group properties by CDN and generate parent.config for each CDN + grouped := make(map[string][]*CustomerConfig) + for _, config := range parsedConfigs { + for cdn := range config.cdn { + grouped[cdn] = append(grouped[cdn], config) + } + } + return grouped +} diff --git a/tools/voluspa/validate.go b/tools/voluspa/validate.go new file mode 100644 index 00000000000..eebb1e189b8 --- /dev/null +++ b/tools/voluspa/validate.go @@ -0,0 +1,312 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package voluspa + +import ( + "errors" + "fmt" + "net" + "net/url" + "sort" + "strings" + + "github.com/apache/trafficserver/tools/voluspa/internal/util/regex" +) + +type Errors []error + +func (err Errors) Error() string { + if len(err) == 0 { + return "Unknown" + } + + var errOut []string + for _, e := range err { + errOut = append(errOut, e.Error()) + } + sort.Strings(errOut) + + return strings.Join(errOut, "\n") +} + +func (err *Errors) Add(newErr error) { + *err = append(*err, newErr) +} + +func (err *Errors) HasErrors() bool { + return len(*err) > 0 +} + +func isValidScheme(scheme string) bool { + return scheme == "http" || scheme == "https" +} + +func (v *Voluspa) ensureValidSchemes() Errors { + var errs Errors + for _, config := range v.parsedConfigs { + for _, scheme := range config.schemes { + if !isValidScheme(scheme) { + errs.Add(fmt.Errorf("property %q contains invalid scheme: %s (%s)", config.property, scheme, config.filename)) + } + } + + for _, remap := range config.Remaps { + if len(remap.scheme) > 0 && !isValidScheme(remap.scheme) { + errs.Add(fmt.Errorf("property %q contains invalid scheme: %s (%s)", config.property, remap.scheme, config.filename)) + } + + } + } + if len(errs) > 0 { + return errs + } + return nil +} + +func (v *Voluspa) ensurePropertyNameUniqueness() Errors { + var errs Errors + + properties := make(map[string]interface{}) + + for _, config := range v.parsedConfigs { + if _, exists := properties[config.property]; exists { + errs.Add(fmt.Errorf("property %s not unique (%s)", config.property, config.filename)) + } else { + properties[config.property] = true + } + } + + if len(errs) > 0 { + return errs + } + return nil +} + +func (v *Voluspa) ensureAliasUniqueness() Errors { + var errs Errors + // map of cdn to role to configlocation to alias + aliases := make(map[string]map[string]map[ConfigLocation]map[string]interface{}) + + // Initialize map/struct + for _, config := range v.parsedConfigs { + for cdn := range config.cdn { + if _, exists := aliases[cdn]; !exists { + aliases[cdn] = make(map[string]map[ConfigLocation]map[string]interface{}) + } + if _, exists := aliases[cdn][config.role]; !exists { + aliases[cdn][config.role] = make(map[ConfigLocation]map[string]interface{}) + for _, location := range []ConfigLocation{UnknownLocation, ChildConfig, ParentConfig} { + aliases[cdn][config.role][location] = make(map[string]interface{}) + } + } + } + } + + for _, config := range v.parsedConfigs { + for cdn := range config.cdn { + for _, remap := range config.Remaps { + alias := remap.IncomingURL + + // aliases are unique based on the aliases itself + the group it's part of and the role + aliasKey := alias + remap.Group + remap.Role + + if _, exists := aliases[cdn][config.role][remap.ConfigLocation][aliasKey]; exists { + var err error + switch { + case len(config.role) > 0: + err = fmt.Errorf("alias %s not unique for role %q (%s)", alias, config.role, config.filename) + case len(remap.Role) > 0: + err = fmt.Errorf("alias %s not unique for rule role %q (%s)", alias, remap.Role, config.filename) + default: + err = fmt.Errorf("alias %s not unique (%s)", alias, config.filename) + } + errs.Add(err) + } else { + aliases[cdn][config.role][remap.ConfigLocation][aliasKey] = nil + } + } + } + } + + if len(errs) > 0 { + return errs + } + return nil +} + +func (v *Voluspa) ensureHostsResolve() Errors { + var errs Errors + + for _, config := range v.parsedConfigs { + for _, remap := range config.Remaps { + if remap.RegexMap { + continue + } + + url, err := url.Parse(remap.OriginURL) + if err != nil { + errs.Add(fmt.Errorf("%s invalid url; %s (%s)", remap.OriginURL, err, config.filename)) + } + if err = isResolvable(url.Host); err != nil { + errs = append(errs, fmt.Errorf("%s unresolvable host (%s)", url.Host, config.filename)) + } + } + } + + if len(errs) > 0 { + return errs + } + return nil +} + +func isResolvable(host string) error { + if strings.Contains(host, ":") { + host = strings.Split(host, ":")[0] + } + + _, err := net.LookupHost(host) + return err +} + +// See https://docs.trafficserver.apache.org/en/latest/admin-guide/plugins/cachekey.en.html#capture-definition +// This is either a regex or a "/" + regex_with_capture + "/" + regex_replace + "/" +func validateCacheKeyCapture(input string) error { + + validator := ®ex.GoLangRegex{} + + regexes := make([]string, 0) + currentRe := "" + prev := ' ' + + // loop over all the characters + for _, c := range input { + if c == '/' && prev != '\\' { // find an unescaped / - it is the boundary + if currentRe != "" { + regexes = append(regexes, currentRe) + } + currentRe = "" + prev = c + continue + } + currentRe += string(c) + prev = c + } + + // if nothing was pushed, there was no / at all, so push the whole string and a dummy + if currentRe != "" { + regexes = append(regexes, currentRe) + regexes = append(regexes, "dummy") + } + + // too many unescaped /'s + if len(regexes) != 2 { + return errors.New("unexpected /") + } + + // check the regexes + for _, re := range regexes { + _, err := validator.IsValid(re) + if err != nil { + return err + } + } + return nil +} + +func (v *Voluspa) validateRegexOptions() Errors { + validator := ®ex.GoLangRegex{} + + var errs Errors + for _, config := range v.parsedConfigs { + for _, remap := range config.Remaps { + if remap.RegexMap { + _, err := validator.IsValid(remap.IncomingURL) + if err != nil { + errs = append(errs, fmt.Errorf("invalid regex URL: %s (%s)", err, config.filename)) + } + } + + for k := range remap.sourceConfig.RemapOptions { + switch k { + case "header_rewrite", // HANDLE header_rewrite separately (split and look for Cond) + "regex_remap", + "asset_token_include", + "asset_token_exclude": + + v, err := remap.sourceConfig.RemapOptions.ValueByNameAsString(k) + if err != nil { + errs = append(errs, fmt.Errorf("%s invalid field. err=%s (%s)", k, err, config.filename)) + continue + } + + _, err = validator.IsValid(v) + if err != nil { + errs = append(errs, fmt.Errorf("invalid regex: %s (%s)", err, config.filename)) + } + + case "content_type_forge": + forgeMap, err := remap.sourceConfig.RemapOptions.ValueByNameAsStringMapString(k) + if err != nil { + errs = append(errs, fmt.Errorf("%s invalid field. err=%s (%s)", k, err, config.filename)) + continue + } + + for mapKey := range forgeMap { + _, err := validator.IsValid(mapKey) + if err != nil { + errs = append(errs, fmt.Errorf("invalid regex: %s (%s)", err, config.filename)) + } + } + case "cachekey": + cachekeyMap, err := remap.sourceConfig.RemapOptions.ValueByNameAsStringMapInterface(k) + if err != nil { + errs = append(errs, fmt.Errorf("%s invalid field. err=%s (%s)", k, err, config.filename)) + continue + } + + for mapKey, mapValue := range cachekeyMap { + if mapKey == "regex_replace_path" || mapKey == "regex_replace_path_uri" || mapKey == "capture_path" { + err := validateCacheKeyCapture(mapValue.(string)) + if err != nil { + errs = append(errs, fmt.Errorf("invalid regex: %s (%s)", err, config.filename)) + } + } + if mapKey == "capture_header" { + for _, re := range mapValue.([]interface{}) { + regexParts := strings.Split(re.(string), ":")[1:] + regexString := strings.Join(regexParts, ":") + err := validateCacheKeyCapture(regexString) + if err != nil { + errs = append(errs, fmt.Errorf("invalid regex: %s (%s)", err, config.filename)) + } + + } + } + } + } + } + } + } + + if len(errs) > 0 { + return errs + } + return nil +} diff --git a/tools/voluspa/vendor/.gitignore b/tools/voluspa/vendor/.gitignore new file mode 100644 index 00000000000..7fa54591b3a --- /dev/null +++ b/tools/voluspa/vendor/.gitignore @@ -0,0 +1,9 @@ +github.com/hpcloud/tail/vendor +.travis* +README* +CONTRIB* +*sh +*pl +*conf +Gopkg.* +*yaml diff --git a/tools/voluspa/vendor/github.com/xeipuuv/gojsonpointer/LICENSE-APACHE-2.0.txt b/tools/voluspa/vendor/github.com/xeipuuv/gojsonpointer/LICENSE-APACHE-2.0.txt new file mode 100644 index 00000000000..55ede8a42cc --- /dev/null +++ b/tools/voluspa/vendor/github.com/xeipuuv/gojsonpointer/LICENSE-APACHE-2.0.txt @@ -0,0 +1,202 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright 2015 xeipuuv + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/tools/voluspa/vendor/github.com/xeipuuv/gojsonpointer/pointer.go b/tools/voluspa/vendor/github.com/xeipuuv/gojsonpointer/pointer.go new file mode 100644 index 00000000000..7faf5d7f943 --- /dev/null +++ b/tools/voluspa/vendor/github.com/xeipuuv/gojsonpointer/pointer.go @@ -0,0 +1,211 @@ +// Copyright 2015 xeipuuv ( https://github.com/xeipuuv ) +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// author xeipuuv +// author-github https://github.com/xeipuuv +// author-mail xeipuuv@gmail.com +// +// repository-name gojsonpointer +// repository-desc An implementation of JSON Pointer - Go language +// +// description Main and unique file. +// +// created 25-02-2013 + +package gojsonpointer + +import ( + "errors" + "fmt" + "reflect" + "strconv" + "strings" +) + +const ( + const_empty_pointer = `` + const_pointer_separator = `/` + + const_invalid_start = `JSON pointer must be empty or start with a "` + const_pointer_separator + `"` +) + +type implStruct struct { + mode string // "SET" or "GET" + + inDocument interface{} + + setInValue interface{} + + getOutNode interface{} + getOutKind reflect.Kind + outError error +} + +type JsonPointer struct { + referenceTokens []string +} + +// NewJsonPointer parses the given string JSON pointer and returns an object +func NewJsonPointer(jsonPointerString string) (p JsonPointer, err error) { + + // Pointer to the root of the document + if len(jsonPointerString) == 0 { + // Keep referenceTokens nil + return + } + if jsonPointerString[0] != '/' { + return p, errors.New(const_invalid_start) + } + + p.referenceTokens = strings.Split(jsonPointerString[1:], const_pointer_separator) + return +} + +// Uses the pointer to retrieve a value from a JSON document +func (p *JsonPointer) Get(document interface{}) (interface{}, reflect.Kind, error) { + + is := &implStruct{mode: "GET", inDocument: document} + p.implementation(is) + return is.getOutNode, is.getOutKind, is.outError + +} + +// Uses the pointer to update a value from a JSON document +func (p *JsonPointer) Set(document interface{}, value interface{}) (interface{}, error) { + + is := &implStruct{mode: "SET", inDocument: document, setInValue: value} + p.implementation(is) + return document, is.outError + +} + +// Uses the pointer to delete a value from a JSON document +func (p *JsonPointer) Delete(document interface{}) (interface{}, error) { + is := &implStruct{mode: "DEL", inDocument: document} + p.implementation(is) + return document, is.outError +} + +// Both Get and Set functions use the same implementation to avoid code duplication +func (p *JsonPointer) implementation(i *implStruct) { + + kind := reflect.Invalid + + // Full document when empty + if len(p.referenceTokens) == 0 { + i.getOutNode = i.inDocument + i.outError = nil + i.getOutKind = kind + i.outError = nil + return + } + + node := i.inDocument + + previousNodes := make([]interface{}, len(p.referenceTokens)) + previousTokens := make([]string, len(p.referenceTokens)) + + for ti, token := range p.referenceTokens { + + isLastToken := ti == len(p.referenceTokens)-1 + previousNodes[ti] = node + previousTokens[ti] = token + + switch v := node.(type) { + + case map[string]interface{}: + decodedToken := decodeReferenceToken(token) + if _, ok := v[decodedToken]; ok { + node = v[decodedToken] + if isLastToken && i.mode == "SET" { + v[decodedToken] = i.setInValue + } else if isLastToken && i.mode =="DEL" { + delete(v,decodedToken) + } + } else if (isLastToken && i.mode == "SET") { + v[decodedToken] = i.setInValue + } else { + i.outError = fmt.Errorf("Object has no key '%s'", decodedToken) + i.getOutKind = reflect.Map + i.getOutNode = nil + return + } + + case []interface{}: + tokenIndex, err := strconv.Atoi(token) + if err != nil { + i.outError = fmt.Errorf("Invalid array index '%s'", token) + i.getOutKind = reflect.Slice + i.getOutNode = nil + return + } + if tokenIndex < 0 || tokenIndex >= len(v) { + i.outError = fmt.Errorf("Out of bound array[0,%d] index '%d'", len(v), tokenIndex) + i.getOutKind = reflect.Slice + i.getOutNode = nil + return + } + + node = v[tokenIndex] + if isLastToken && i.mode == "SET" { + v[tokenIndex] = i.setInValue + } else if isLastToken && i.mode =="DEL" { + v[tokenIndex] = v[len(v)-1] + v[len(v)-1] = nil + v = v[:len(v)-1] + previousNodes[ti-1].(map[string]interface{})[previousTokens[ti-1]] = v + } + + default: + i.outError = fmt.Errorf("Invalid token reference '%s'", token) + i.getOutKind = reflect.ValueOf(node).Kind() + i.getOutNode = nil + return + } + + } + + i.getOutNode = node + i.getOutKind = reflect.ValueOf(node).Kind() + i.outError = nil +} + +// Pointer to string representation function +func (p *JsonPointer) String() string { + + if len(p.referenceTokens) == 0 { + return const_empty_pointer + } + + pointerString := const_pointer_separator + strings.Join(p.referenceTokens, const_pointer_separator) + + return pointerString +} + +// Specific JSON pointer encoding here +// ~0 => ~ +// ~1 => / +// ... and vice versa + +func decodeReferenceToken(token string) string { + step1 := strings.Replace(token, `~1`, `/`, -1) + step2 := strings.Replace(step1, `~0`, `~`, -1) + return step2 +} + +func encodeReferenceToken(token string) string { + step1 := strings.Replace(token, `~`, `~0`, -1) + step2 := strings.Replace(step1, `/`, `~1`, -1) + return step2 +} diff --git a/tools/voluspa/vendor/github.com/xeipuuv/gojsonreference/LICENSE-APACHE-2.0.txt b/tools/voluspa/vendor/github.com/xeipuuv/gojsonreference/LICENSE-APACHE-2.0.txt new file mode 100644 index 00000000000..55ede8a42cc --- /dev/null +++ b/tools/voluspa/vendor/github.com/xeipuuv/gojsonreference/LICENSE-APACHE-2.0.txt @@ -0,0 +1,202 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright 2015 xeipuuv + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/tools/voluspa/vendor/github.com/xeipuuv/gojsonreference/reference.go b/tools/voluspa/vendor/github.com/xeipuuv/gojsonreference/reference.go new file mode 100644 index 00000000000..6457291301d --- /dev/null +++ b/tools/voluspa/vendor/github.com/xeipuuv/gojsonreference/reference.go @@ -0,0 +1,147 @@ +// Copyright 2015 xeipuuv ( https://github.com/xeipuuv ) +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// author xeipuuv +// author-github https://github.com/xeipuuv +// author-mail xeipuuv@gmail.com +// +// repository-name gojsonreference +// repository-desc An implementation of JSON Reference - Go language +// +// description Main and unique file. +// +// created 26-02-2013 + +package gojsonreference + +import ( + "errors" + "net/url" + "path/filepath" + "runtime" + "strings" + + "github.com/xeipuuv/gojsonpointer" +) + +const ( + const_fragment_char = `#` +) + +func NewJsonReference(jsonReferenceString string) (JsonReference, error) { + + var r JsonReference + err := r.parse(jsonReferenceString) + return r, err + +} + +type JsonReference struct { + referenceUrl *url.URL + referencePointer gojsonpointer.JsonPointer + + HasFullUrl bool + HasUrlPathOnly bool + HasFragmentOnly bool + HasFileScheme bool + HasFullFilePath bool +} + +func (r *JsonReference) GetUrl() *url.URL { + return r.referenceUrl +} + +func (r *JsonReference) GetPointer() *gojsonpointer.JsonPointer { + return &r.referencePointer +} + +func (r *JsonReference) String() string { + + if r.referenceUrl != nil { + return r.referenceUrl.String() + } + + if r.HasFragmentOnly { + return const_fragment_char + r.referencePointer.String() + } + + return r.referencePointer.String() +} + +func (r *JsonReference) IsCanonical() bool { + return (r.HasFileScheme && r.HasFullFilePath) || (!r.HasFileScheme && r.HasFullUrl) +} + +// "Constructor", parses the given string JSON reference +func (r *JsonReference) parse(jsonReferenceString string) (err error) { + + r.referenceUrl, err = url.Parse(jsonReferenceString) + if err != nil { + return + } + refUrl := r.referenceUrl + + if refUrl.Scheme != "" && refUrl.Host != "" { + r.HasFullUrl = true + } else { + if refUrl.Path != "" { + r.HasUrlPathOnly = true + } else if refUrl.RawQuery == "" && refUrl.Fragment != "" { + r.HasFragmentOnly = true + } + } + + r.HasFileScheme = refUrl.Scheme == "file" + if runtime.GOOS == "windows" { + // on Windows, a file URL may have an extra leading slash, and if it + // doesn't then its first component will be treated as the host by the + // Go runtime + if refUrl.Host == "" && strings.HasPrefix(refUrl.Path, "/") { + r.HasFullFilePath = filepath.IsAbs(refUrl.Path[1:]) + } else { + r.HasFullFilePath = filepath.IsAbs(refUrl.Host + refUrl.Path) + } + } else { + r.HasFullFilePath = filepath.IsAbs(refUrl.Path) + } + + // invalid json-pointer error means url has no json-pointer fragment. simply ignore error + r.referencePointer, _ = gojsonpointer.NewJsonPointer(refUrl.Fragment) + + return +} + +// Creates a new reference from a parent and a child +// If the child cannot inherit from the parent, an error is returned +func (r *JsonReference) Inherits(child JsonReference) (*JsonReference, error) { + if child.GetUrl() == nil { + return nil, errors.New("childUrl is nil!") + } + + if r.GetUrl() == nil { + return nil, errors.New("parentUrl is nil!") + } + + // Get a copy of the parent url to make sure we do not modify the original. + // URL reference resolving fails if the fragment of the child is empty, but the parent's is not. + // The fragment of the child must be used, so the fragment of the parent is manually removed. + parentUrl := *r.GetUrl() + parentUrl.Fragment = "" + + ref, err := NewJsonReference(parentUrl.ResolveReference(child.GetUrl()).String()) + if err != nil { + return nil, err + } + return &ref, err +} diff --git a/tools/voluspa/vendor/github.com/xeipuuv/gojsonschema/.gitignore b/tools/voluspa/vendor/github.com/xeipuuv/gojsonschema/.gitignore new file mode 100644 index 00000000000..68e993ce3e0 --- /dev/null +++ b/tools/voluspa/vendor/github.com/xeipuuv/gojsonschema/.gitignore @@ -0,0 +1,3 @@ +*.sw[nop] +*.iml +.vscode/ diff --git a/tools/voluspa/vendor/github.com/xeipuuv/gojsonschema/LICENSE-APACHE-2.0.txt b/tools/voluspa/vendor/github.com/xeipuuv/gojsonschema/LICENSE-APACHE-2.0.txt new file mode 100644 index 00000000000..55ede8a42cc --- /dev/null +++ b/tools/voluspa/vendor/github.com/xeipuuv/gojsonschema/LICENSE-APACHE-2.0.txt @@ -0,0 +1,202 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright 2015 xeipuuv + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/tools/voluspa/vendor/github.com/xeipuuv/gojsonschema/errors.go b/tools/voluspa/vendor/github.com/xeipuuv/gojsonschema/errors.go new file mode 100644 index 00000000000..2f326a2228b --- /dev/null +++ b/tools/voluspa/vendor/github.com/xeipuuv/gojsonschema/errors.go @@ -0,0 +1,324 @@ +package gojsonschema + +import ( + "bytes" + "sync" + "text/template" +) + +var errorTemplates errorTemplate = errorTemplate{template.New("errors-new"), sync.RWMutex{}} + +// template.Template is not thread-safe for writing, so some locking is done +// sync.RWMutex is used for efficiently locking when new templates are created +type errorTemplate struct { + *template.Template + sync.RWMutex +} + +type ( + // RequiredError. ErrorDetails: property string + RequiredError struct { + ResultErrorFields + } + + // InvalidTypeError. ErrorDetails: expected, given + InvalidTypeError struct { + ResultErrorFields + } + + // NumberAnyOfError. ErrorDetails: - + NumberAnyOfError struct { + ResultErrorFields + } + + // NumberOneOfError. ErrorDetails: - + NumberOneOfError struct { + ResultErrorFields + } + + // NumberAllOfError. ErrorDetails: - + NumberAllOfError struct { + ResultErrorFields + } + + // NumberNotError. ErrorDetails: - + NumberNotError struct { + ResultErrorFields + } + + // MissingDependencyError. ErrorDetails: dependency + MissingDependencyError struct { + ResultErrorFields + } + + // InternalError. ErrorDetails: error + InternalError struct { + ResultErrorFields + } + + // ConstError. ErrorDetails: allowed + ConstError struct { + ResultErrorFields + } + + // EnumError. ErrorDetails: allowed + EnumError struct { + ResultErrorFields + } + + // ArrayNoAdditionalItemsError. ErrorDetails: - + ArrayNoAdditionalItemsError struct { + ResultErrorFields + } + + // ArrayMinItemsError. ErrorDetails: min + ArrayMinItemsError struct { + ResultErrorFields + } + + // ArrayMaxItemsError. ErrorDetails: max + ArrayMaxItemsError struct { + ResultErrorFields + } + + // ItemsMustBeUniqueError. ErrorDetails: type + ItemsMustBeUniqueError struct { + ResultErrorFields + } + + // ArrayContainsError. ErrorDetails: + ArrayContainsError struct { + ResultErrorFields + } + + // ArrayMinPropertiesError. ErrorDetails: min + ArrayMinPropertiesError struct { + ResultErrorFields + } + + // ArrayMaxPropertiesError. ErrorDetails: max + ArrayMaxPropertiesError struct { + ResultErrorFields + } + + // AdditionalPropertyNotAllowedError. ErrorDetails: property + AdditionalPropertyNotAllowedError struct { + ResultErrorFields + } + + // InvalidPropertyPatternError. ErrorDetails: property, pattern + InvalidPropertyPatternError struct { + ResultErrorFields + } + + // InvalidPopertyNameError. ErrorDetails: property + InvalidPropertyNameError struct { + ResultErrorFields + } + + // StringLengthGTEError. ErrorDetails: min + StringLengthGTEError struct { + ResultErrorFields + } + + // StringLengthLTEError. ErrorDetails: max + StringLengthLTEError struct { + ResultErrorFields + } + + // DoesNotMatchPatternError. ErrorDetails: pattern + DoesNotMatchPatternError struct { + ResultErrorFields + } + + // DoesNotMatchFormatError. ErrorDetails: format + DoesNotMatchFormatError struct { + ResultErrorFields + } + + // MultipleOfError. ErrorDetails: multiple + MultipleOfError struct { + ResultErrorFields + } + + // NumberGTEError. ErrorDetails: min + NumberGTEError struct { + ResultErrorFields + } + + // NumberGTError. ErrorDetails: min + NumberGTError struct { + ResultErrorFields + } + + // NumberLTEError. ErrorDetails: max + NumberLTEError struct { + ResultErrorFields + } + + // NumberLTError. ErrorDetails: max + NumberLTError struct { + ResultErrorFields + } + + // ConditionThenError. ErrorDetails: - + ConditionThenError struct { + ResultErrorFields + } + + // ConditionElseError. ErrorDetails: - + ConditionElseError struct { + ResultErrorFields + } +) + +// newError takes a ResultError type and sets the type, context, description, details, value, and field +func newError(err ResultError, context *JsonContext, value interface{}, locale locale, details ErrorDetails) { + var t string + var d string + switch err.(type) { + case *RequiredError: + t = "required" + d = locale.Required() + case *InvalidTypeError: + t = "invalid_type" + d = locale.InvalidType() + case *NumberAnyOfError: + t = "number_any_of" + d = locale.NumberAnyOf() + case *NumberOneOfError: + t = "number_one_of" + d = locale.NumberOneOf() + case *NumberAllOfError: + t = "number_all_of" + d = locale.NumberAllOf() + case *NumberNotError: + t = "number_not" + d = locale.NumberNot() + case *MissingDependencyError: + t = "missing_dependency" + d = locale.MissingDependency() + case *InternalError: + t = "internal" + d = locale.Internal() + case *ConstError: + t = "const" + d = locale.Const() + case *EnumError: + t = "enum" + d = locale.Enum() + case *ArrayNoAdditionalItemsError: + t = "array_no_additional_items" + d = locale.ArrayNoAdditionalItems() + case *ArrayMinItemsError: + t = "array_min_items" + d = locale.ArrayMinItems() + case *ArrayMaxItemsError: + t = "array_max_items" + d = locale.ArrayMaxItems() + case *ItemsMustBeUniqueError: + t = "unique" + d = locale.Unique() + case *ArrayContainsError: + t = "contains" + d = locale.ArrayContains() + case *ArrayMinPropertiesError: + t = "array_min_properties" + d = locale.ArrayMinProperties() + case *ArrayMaxPropertiesError: + t = "array_max_properties" + d = locale.ArrayMaxProperties() + case *AdditionalPropertyNotAllowedError: + t = "additional_property_not_allowed" + d = locale.AdditionalPropertyNotAllowed() + case *InvalidPropertyPatternError: + t = "invalid_property_pattern" + d = locale.InvalidPropertyPattern() + case *InvalidPropertyNameError: + t = "invalid_property_name" + d = locale.InvalidPropertyName() + case *StringLengthGTEError: + t = "string_gte" + d = locale.StringGTE() + case *StringLengthLTEError: + t = "string_lte" + d = locale.StringLTE() + case *DoesNotMatchPatternError: + t = "pattern" + d = locale.DoesNotMatchPattern() + case *DoesNotMatchFormatError: + t = "format" + d = locale.DoesNotMatchFormat() + case *MultipleOfError: + t = "multiple_of" + d = locale.MultipleOf() + case *NumberGTEError: + t = "number_gte" + d = locale.NumberGTE() + case *NumberGTError: + t = "number_gt" + d = locale.NumberGT() + case *NumberLTEError: + t = "number_lte" + d = locale.NumberLTE() + case *NumberLTError: + t = "number_lt" + d = locale.NumberLT() + case *ConditionThenError: + t = "condition_then" + d = locale.ConditionThen() + case *ConditionElseError: + t = "condition_else" + d = locale.ConditionElse() + } + + err.SetType(t) + err.SetContext(context) + err.SetValue(value) + err.SetDetails(details) + err.SetDescriptionFormat(d) + details["field"] = err.Field() + + if _, exists := details["context"]; !exists && context != nil { + details["context"] = context.String() + } + + err.SetDescription(formatErrorDescription(err.DescriptionFormat(), details)) +} + +// formatErrorDescription takes a string in the default text/template +// format and converts it to a string with replacements. The fields come +// from the ErrorDetails struct and vary for each type of error. +func formatErrorDescription(s string, details ErrorDetails) string { + + var tpl *template.Template + var descrAsBuffer bytes.Buffer + var err error + + errorTemplates.RLock() + tpl = errorTemplates.Lookup(s) + errorTemplates.RUnlock() + + if tpl == nil { + errorTemplates.Lock() + tpl = errorTemplates.New(s) + + if ErrorTemplateFuncs != nil { + tpl.Funcs(ErrorTemplateFuncs) + } + + tpl, err = tpl.Parse(s) + errorTemplates.Unlock() + + if err != nil { + return err.Error() + } + } + + err = tpl.Execute(&descrAsBuffer, details) + if err != nil { + return err.Error() + } + + return descrAsBuffer.String() +} diff --git a/tools/voluspa/vendor/github.com/xeipuuv/gojsonschema/format_checkers.go b/tools/voluspa/vendor/github.com/xeipuuv/gojsonschema/format_checkers.go new file mode 100644 index 00000000000..26217fca128 --- /dev/null +++ b/tools/voluspa/vendor/github.com/xeipuuv/gojsonschema/format_checkers.go @@ -0,0 +1,343 @@ +package gojsonschema + +import ( + "net" + "net/mail" + "net/url" + "regexp" + "strings" + "sync" + "time" +) + +type ( + // FormatChecker is the interface all formatters added to FormatCheckerChain must implement + FormatChecker interface { + IsFormat(input interface{}) bool + } + + // FormatCheckerChain holds the formatters + FormatCheckerChain struct { + formatters map[string]FormatChecker + } + + // EmailFormatter verifies email address formats + EmailFormatChecker struct{} + + // IPV4FormatChecker verifies IP addresses in the ipv4 format + IPV4FormatChecker struct{} + + // IPV6FormatChecker verifies IP addresses in the ipv6 format + IPV6FormatChecker struct{} + + // DateTimeFormatChecker verifies date/time formats per RFC3339 5.6 + // + // Valid formats: + // Partial Time: HH:MM:SS + // Full Date: YYYY-MM-DD + // Full Time: HH:MM:SSZ-07:00 + // Date Time: YYYY-MM-DDTHH:MM:SSZ-0700 + // + // Where + // YYYY = 4DIGIT year + // MM = 2DIGIT month ; 01-12 + // DD = 2DIGIT day-month ; 01-28, 01-29, 01-30, 01-31 based on month/year + // HH = 2DIGIT hour ; 00-23 + // MM = 2DIGIT ; 00-59 + // SS = 2DIGIT ; 00-58, 00-60 based on leap second rules + // T = Literal + // Z = Literal + // + // Note: Nanoseconds are also suported in all formats + // + // http://tools.ietf.org/html/rfc3339#section-5.6 + DateTimeFormatChecker struct{} + + DateFormatChecker struct{} + + TimeFormatChecker struct{} + + // URIFormatChecker validates a URI with a valid Scheme per RFC3986 + URIFormatChecker struct{} + + // URIReferenceFormatChecker validates a URI or relative-reference per RFC3986 + URIReferenceFormatChecker struct{} + + // URITemplateFormatChecker validates a URI template per RFC6570 + URITemplateFormatChecker struct{} + + // HostnameFormatChecker validates a hostname is in the correct format + HostnameFormatChecker struct{} + + // UUIDFormatChecker validates a UUID is in the correct format + UUIDFormatChecker struct{} + + // RegexFormatChecker validates a regex is in the correct format + RegexFormatChecker struct{} + + // JSONPointerFormatChecker validates a JSON Pointer per RFC6901 + JSONPointerFormatChecker struct{} + + // RelativeJSONPointerFormatChecker validates a relative JSON Pointer is in the correct format + RelativeJSONPointerFormatChecker struct{} +) + +var ( + // Formatters holds the valid formatters, and is a public variable + // so library users can add custom formatters + FormatCheckers = FormatCheckerChain{ + formatters: map[string]FormatChecker{ + "date": DateFormatChecker{}, + "time": TimeFormatChecker{}, + "date-time": DateTimeFormatChecker{}, + "hostname": HostnameFormatChecker{}, + "email": EmailFormatChecker{}, + "idn-email": EmailFormatChecker{}, + "ipv4": IPV4FormatChecker{}, + "ipv6": IPV6FormatChecker{}, + "uri": URIFormatChecker{}, + "uri-reference": URIReferenceFormatChecker{}, + "iri": URIFormatChecker{}, + "iri-reference": URIReferenceFormatChecker{}, + "uri-template": URITemplateFormatChecker{}, + "uuid": UUIDFormatChecker{}, + "regex": RegexFormatChecker{}, + "json-pointer": JSONPointerFormatChecker{}, + "relative-json-pointer": RelativeJSONPointerFormatChecker{}, + }, + } + + // Regex credit: https://www.socketloop.com/tutorials/golang-validate-hostname + rxHostname = regexp.MustCompile(`^([a-zA-Z0-9]|[a-zA-Z0-9][a-zA-Z0-9\-]{0,61}[a-zA-Z0-9])(\.([a-zA-Z0-9]|[a-zA-Z0-9][a-zA-Z0-9\-]{0,61}[a-zA-Z0-9]))*$`) + + // Use a regex to make sure curly brackets are balanced properly after validating it as a AURI + rxURITemplate = regexp.MustCompile("^([^{]*({[^}]*})?)*$") + + rxUUID = regexp.MustCompile("^[a-f0-9]{8}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{12}$") + + rxJSONPointer = regexp.MustCompile("^(?:/(?:[^~/]|~0|~1)*)*$") + + rxRelJSONPointer = regexp.MustCompile("^(?:0|[1-9][0-9]*)(?:#|(?:/(?:[^~/]|~0|~1)*)*)$") + + lock = new(sync.Mutex) +) + +// Add adds a FormatChecker to the FormatCheckerChain +// The name used will be the value used for the format key in your json schema +func (c *FormatCheckerChain) Add(name string, f FormatChecker) *FormatCheckerChain { + lock.Lock() + c.formatters[name] = f + lock.Unlock() + + return c +} + +// Remove deletes a FormatChecker from the FormatCheckerChain (if it exists) +func (c *FormatCheckerChain) Remove(name string) *FormatCheckerChain { + lock.Lock() + delete(c.formatters, name) + lock.Unlock() + + return c +} + +// Has checks to see if the FormatCheckerChain holds a FormatChecker with the given name +func (c *FormatCheckerChain) Has(name string) bool { + lock.Lock() + _, ok := c.formatters[name] + lock.Unlock() + + return ok +} + +// IsFormat will check an input against a FormatChecker with the given name +// to see if it is the correct format +func (c *FormatCheckerChain) IsFormat(name string, input interface{}) bool { + f, ok := c.formatters[name] + + if !ok { + return false + } + + return f.IsFormat(input) +} + +func (f EmailFormatChecker) IsFormat(input interface{}) bool { + + asString, ok := input.(string) + if ok == false { + return false + } + + _, err := mail.ParseAddress(asString) + + return err == nil +} + +// Credit: https://github.com/asaskevich/govalidator +func (f IPV4FormatChecker) IsFormat(input interface{}) bool { + + asString, ok := input.(string) + if ok == false { + return false + } + + ip := net.ParseIP(asString) + return ip != nil && strings.Contains(asString, ".") +} + +// Credit: https://github.com/asaskevich/govalidator +func (f IPV6FormatChecker) IsFormat(input interface{}) bool { + + asString, ok := input.(string) + if ok == false { + return false + } + + ip := net.ParseIP(asString) + return ip != nil && strings.Contains(asString, ":") +} + +func (f DateTimeFormatChecker) IsFormat(input interface{}) bool { + + asString, ok := input.(string) + if ok == false { + return false + } + + formats := []string{ + "15:04:05", + "15:04:05Z07:00", + "2006-01-02", + time.RFC3339, + time.RFC3339Nano, + } + + for _, format := range formats { + if _, err := time.Parse(format, asString); err == nil { + return true + } + } + + return false +} + +func (f DateFormatChecker) IsFormat(input interface{}) bool { + asString, ok := input.(string) + if ok == false { + return false + } + _, err := time.Parse("2006-01-02", asString) + return err == nil +} + +func (f TimeFormatChecker) IsFormat(input interface{}) bool { + asString, ok := input.(string) + if ok == false { + return false + } + + if _, err := time.Parse("15:04:05Z07:00", asString); err == nil { + return true + } + + _, err := time.Parse("15:04:05", asString) + return err == nil +} + +func (f URIFormatChecker) IsFormat(input interface{}) bool { + + asString, ok := input.(string) + if ok == false { + return false + } + + u, err := url.Parse(asString) + + if err != nil || u.Scheme == "" { + return false + } + + return !strings.Contains(asString, `\`) +} + +func (f URIReferenceFormatChecker) IsFormat(input interface{}) bool { + + asString, ok := input.(string) + if ok == false { + return false + } + + _, err := url.Parse(asString) + return err == nil && !strings.Contains(asString, `\`) +} + +func (f URITemplateFormatChecker) IsFormat(input interface{}) bool { + asString, ok := input.(string) + if ok == false { + return false + } + + u, err := url.Parse(asString) + if err != nil || strings.Contains(asString, `\`) { + return false + } + + return rxURITemplate.MatchString(u.Path) +} + +func (f HostnameFormatChecker) IsFormat(input interface{}) bool { + + asString, ok := input.(string) + if ok == false { + return false + } + + return rxHostname.MatchString(asString) && len(asString) < 256 +} + +func (f UUIDFormatChecker) IsFormat(input interface{}) bool { + + asString, ok := input.(string) + if ok == false { + return false + } + + return rxUUID.MatchString(asString) +} + +// IsFormat implements FormatChecker interface. +func (f RegexFormatChecker) IsFormat(input interface{}) bool { + + asString, ok := input.(string) + if ok == false { + return false + } + + if asString == "" { + return true + } + _, err := regexp.Compile(asString) + if err != nil { + return false + } + return true +} + +func (f JSONPointerFormatChecker) IsFormat(input interface{}) bool { + asString, ok := input.(string) + if ok == false { + return false + } + + return rxJSONPointer.MatchString(asString) +} + +func (f RelativeJSONPointerFormatChecker) IsFormat(input interface{}) bool { + asString, ok := input.(string) + if ok == false { + return false + } + + return rxRelJSONPointer.MatchString(asString) +} diff --git a/tools/voluspa/vendor/github.com/xeipuuv/gojsonschema/internalLog.go b/tools/voluspa/vendor/github.com/xeipuuv/gojsonschema/internalLog.go new file mode 100644 index 00000000000..4ef7a8d03e7 --- /dev/null +++ b/tools/voluspa/vendor/github.com/xeipuuv/gojsonschema/internalLog.go @@ -0,0 +1,37 @@ +// Copyright 2015 xeipuuv ( https://github.com/xeipuuv ) +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// author xeipuuv +// author-github https://github.com/xeipuuv +// author-mail xeipuuv@gmail.com +// +// repository-name gojsonschema +// repository-desc An implementation of JSON Schema, based on IETF's draft v4 - Go language. +// +// description Very simple log wrapper. +// Used for debugging/testing purposes. +// +// created 01-01-2015 + +package gojsonschema + +import ( + "log" +) + +const internalLogEnabled = false + +func internalLog(format string, v ...interface{}) { + log.Printf(format, v...) +} diff --git a/tools/voluspa/vendor/github.com/xeipuuv/gojsonschema/jsonContext.go b/tools/voluspa/vendor/github.com/xeipuuv/gojsonschema/jsonContext.go new file mode 100644 index 00000000000..f40668a74c4 --- /dev/null +++ b/tools/voluspa/vendor/github.com/xeipuuv/gojsonschema/jsonContext.go @@ -0,0 +1,72 @@ +// Copyright 2013 MongoDB, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// author tolsen +// author-github https://github.com/tolsen +// +// repository-name gojsonschema +// repository-desc An implementation of JSON Schema, based on IETF's draft v4 - Go language. +// +// description Implements a persistent (immutable w/ shared structure) singly-linked list of strings for the purpose of storing a json context +// +// created 04-09-2013 + +package gojsonschema + +import "bytes" + +// JsonContext implements a persistent linked-list of strings +type JsonContext struct { + head string + tail *JsonContext +} + +func NewJsonContext(head string, tail *JsonContext) *JsonContext { + return &JsonContext{head, tail} +} + +// String displays the context in reverse. +// This plays well with the data structure's persistent nature with +// Cons and a json document's tree structure. +func (c *JsonContext) String(del ...string) string { + byteArr := make([]byte, 0, c.stringLen()) + buf := bytes.NewBuffer(byteArr) + c.writeStringToBuffer(buf, del) + + return buf.String() +} + +func (c *JsonContext) stringLen() int { + length := 0 + if c.tail != nil { + length = c.tail.stringLen() + 1 // add 1 for "." + } + + length += len(c.head) + return length +} + +func (c *JsonContext) writeStringToBuffer(buf *bytes.Buffer, del []string) { + if c.tail != nil { + c.tail.writeStringToBuffer(buf, del) + + if len(del) > 0 { + buf.WriteString(del[0]) + } else { + buf.WriteString(".") + } + } + + buf.WriteString(c.head) +} diff --git a/tools/voluspa/vendor/github.com/xeipuuv/gojsonschema/jsonLoader.go b/tools/voluspa/vendor/github.com/xeipuuv/gojsonschema/jsonLoader.go new file mode 100644 index 00000000000..cbe46649218 --- /dev/null +++ b/tools/voluspa/vendor/github.com/xeipuuv/gojsonschema/jsonLoader.go @@ -0,0 +1,362 @@ +// Copyright 2015 xeipuuv ( https://github.com/xeipuuv ) +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// author xeipuuv +// author-github https://github.com/xeipuuv +// author-mail xeipuuv@gmail.com +// +// repository-name gojsonschema +// repository-desc An implementation of JSON Schema, based on IETF's draft v4 - Go language. +// +// description Different strategies to load JSON files. +// Includes References (file and HTTP), JSON strings and Go types. +// +// created 01-02-2015 + +package gojsonschema + +import ( + "bytes" + "encoding/json" + "errors" + "io" + "io/ioutil" + "net/http" + "os" + "path/filepath" + "runtime" + "strings" + + "github.com/xeipuuv/gojsonreference" +) + +var osFS = osFileSystem(os.Open) + +// JSON loader interface + +type JSONLoader interface { + JsonSource() interface{} + LoadJSON() (interface{}, error) + JsonReference() (gojsonreference.JsonReference, error) + LoaderFactory() JSONLoaderFactory +} + +type JSONLoaderFactory interface { + New(source string) JSONLoader +} + +type DefaultJSONLoaderFactory struct { +} + +type FileSystemJSONLoaderFactory struct { + fs http.FileSystem +} + +func (d DefaultJSONLoaderFactory) New(source string) JSONLoader { + return &jsonReferenceLoader{ + fs: osFS, + source: source, + } +} + +func (f FileSystemJSONLoaderFactory) New(source string) JSONLoader { + return &jsonReferenceLoader{ + fs: f.fs, + source: source, + } +} + +// osFileSystem is a functional wrapper for os.Open that implements http.FileSystem. +type osFileSystem func(string) (*os.File, error) + +func (o osFileSystem) Open(name string) (http.File, error) { + return o(name) +} + +// JSON Reference loader +// references are used to load JSONs from files and HTTP + +type jsonReferenceLoader struct { + fs http.FileSystem + source string +} + +func (l *jsonReferenceLoader) JsonSource() interface{} { + return l.source +} + +func (l *jsonReferenceLoader) JsonReference() (gojsonreference.JsonReference, error) { + return gojsonreference.NewJsonReference(l.JsonSource().(string)) +} + +func (l *jsonReferenceLoader) LoaderFactory() JSONLoaderFactory { + return &FileSystemJSONLoaderFactory{ + fs: l.fs, + } +} + +// NewReferenceLoader returns a JSON reference loader using the given source and the local OS file system. +func NewReferenceLoader(source string) JSONLoader { + return &jsonReferenceLoader{ + fs: osFS, + source: source, + } +} + +// NewReferenceLoaderFileSystem returns a JSON reference loader using the given source and file system. +func NewReferenceLoaderFileSystem(source string, fs http.FileSystem) JSONLoader { + return &jsonReferenceLoader{ + fs: fs, + source: source, + } +} + +func (l *jsonReferenceLoader) LoadJSON() (interface{}, error) { + + var err error + + reference, err := gojsonreference.NewJsonReference(l.JsonSource().(string)) + if err != nil { + return nil, err + } + + refToUrl := reference + refToUrl.GetUrl().Fragment = "" + + var document interface{} + + if reference.HasFileScheme { + + filename := strings.TrimPrefix(refToUrl.String(), "file://") + if runtime.GOOS == "windows" { + // on Windows, a file URL may have an extra leading slash, use slashes + // instead of backslashes, and have spaces escaped + filename = strings.TrimPrefix(filename, "/") + filename = filepath.FromSlash(filename) + } + + document, err = l.loadFromFile(filename) + if err != nil { + return nil, err + } + + } else { + + document, err = l.loadFromHTTP(refToUrl.String()) + if err != nil { + return nil, err + } + + } + + return document, nil + +} + +func (l *jsonReferenceLoader) loadFromHTTP(address string) (interface{}, error) { + + resp, err := http.Get(address) + if err != nil { + return nil, err + } + + // must return HTTP Status 200 OK + if resp.StatusCode != http.StatusOK { + return nil, errors.New(formatErrorDescription(Locale.HttpBadStatus(), ErrorDetails{"status": resp.Status})) + } + + bodyBuff, err := ioutil.ReadAll(resp.Body) + if err != nil { + return nil, err + } + + return decodeJsonUsingNumber(bytes.NewReader(bodyBuff)) + +} + +func (l *jsonReferenceLoader) loadFromFile(path string) (interface{}, error) { + f, err := l.fs.Open(path) + if err != nil { + return nil, err + } + defer f.Close() + + bodyBuff, err := ioutil.ReadAll(f) + if err != nil { + return nil, err + } + + return decodeJsonUsingNumber(bytes.NewReader(bodyBuff)) + +} + +// JSON string loader + +type jsonStringLoader struct { + source string +} + +func (l *jsonStringLoader) JsonSource() interface{} { + return l.source +} + +func (l *jsonStringLoader) JsonReference() (gojsonreference.JsonReference, error) { + return gojsonreference.NewJsonReference("#") +} + +func (l *jsonStringLoader) LoaderFactory() JSONLoaderFactory { + return &DefaultJSONLoaderFactory{} +} + +func NewStringLoader(source string) JSONLoader { + return &jsonStringLoader{source: source} +} + +func (l *jsonStringLoader) LoadJSON() (interface{}, error) { + + return decodeJsonUsingNumber(strings.NewReader(l.JsonSource().(string))) + +} + +// JSON bytes loader + +type jsonBytesLoader struct { + source []byte +} + +func (l *jsonBytesLoader) JsonSource() interface{} { + return l.source +} + +func (l *jsonBytesLoader) JsonReference() (gojsonreference.JsonReference, error) { + return gojsonreference.NewJsonReference("#") +} + +func (l *jsonBytesLoader) LoaderFactory() JSONLoaderFactory { + return &DefaultJSONLoaderFactory{} +} + +func NewBytesLoader(source []byte) JSONLoader { + return &jsonBytesLoader{source: source} +} + +func (l *jsonBytesLoader) LoadJSON() (interface{}, error) { + return decodeJsonUsingNumber(bytes.NewReader(l.JsonSource().([]byte))) +} + +// JSON Go (types) loader +// used to load JSONs from the code as maps, interface{}, structs ... + +type jsonGoLoader struct { + source interface{} +} + +func (l *jsonGoLoader) JsonSource() interface{} { + return l.source +} + +func (l *jsonGoLoader) JsonReference() (gojsonreference.JsonReference, error) { + return gojsonreference.NewJsonReference("#") +} + +func (l *jsonGoLoader) LoaderFactory() JSONLoaderFactory { + return &DefaultJSONLoaderFactory{} +} + +func NewGoLoader(source interface{}) JSONLoader { + return &jsonGoLoader{source: source} +} + +func (l *jsonGoLoader) LoadJSON() (interface{}, error) { + + // convert it to a compliant JSON first to avoid types "mismatches" + + jsonBytes, err := json.Marshal(l.JsonSource()) + if err != nil { + return nil, err + } + + return decodeJsonUsingNumber(bytes.NewReader(jsonBytes)) + +} + +type jsonIOLoader struct { + buf *bytes.Buffer +} + +func NewReaderLoader(source io.Reader) (JSONLoader, io.Reader) { + buf := &bytes.Buffer{} + return &jsonIOLoader{buf: buf}, io.TeeReader(source, buf) +} + +func NewWriterLoader(source io.Writer) (JSONLoader, io.Writer) { + buf := &bytes.Buffer{} + return &jsonIOLoader{buf: buf}, io.MultiWriter(source, buf) +} + +func (l *jsonIOLoader) JsonSource() interface{} { + return l.buf.String() +} + +func (l *jsonIOLoader) LoadJSON() (interface{}, error) { + return decodeJsonUsingNumber(l.buf) +} + +func (l *jsonIOLoader) JsonReference() (gojsonreference.JsonReference, error) { + return gojsonreference.NewJsonReference("#") +} + +func (l *jsonIOLoader) LoaderFactory() JSONLoaderFactory { + return &DefaultJSONLoaderFactory{} +} + +// JSON raw loader +// In case the JSON is already marshalled to interface{} use this loader +// This is used for testing as otherwise there is no guarantee the JSON is marshalled +// "properly" by using https://golang.org/pkg/encoding/json/#Decoder.UseNumber +type jsonRawLoader struct { + source interface{} +} + +func NewRawLoader(source interface{}) *jsonRawLoader { + return &jsonRawLoader{source: source} +} +func (l *jsonRawLoader) JsonSource() interface{} { + return l.source +} +func (l *jsonRawLoader) LoadJSON() (interface{}, error) { + return l.source, nil +} +func (l *jsonRawLoader) JsonReference() (gojsonreference.JsonReference, error) { + return gojsonreference.NewJsonReference("#") +} +func (l *jsonRawLoader) LoaderFactory() JSONLoaderFactory { + return &DefaultJSONLoaderFactory{} +} + +func decodeJsonUsingNumber(r io.Reader) (interface{}, error) { + + var document interface{} + + decoder := json.NewDecoder(r) + decoder.UseNumber() + + err := decoder.Decode(&document) + if err != nil { + return nil, err + } + + return document, nil + +} diff --git a/tools/voluspa/vendor/github.com/xeipuuv/gojsonschema/locales.go b/tools/voluspa/vendor/github.com/xeipuuv/gojsonschema/locales.go new file mode 100644 index 00000000000..7bd9a9dce59 --- /dev/null +++ b/tools/voluspa/vendor/github.com/xeipuuv/gojsonschema/locales.go @@ -0,0 +1,313 @@ +// Copyright 2015 xeipuuv ( https://github.com/xeipuuv ) +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// author xeipuuv +// author-github https://github.com/xeipuuv +// author-mail xeipuuv@gmail.com +// +// repository-name gojsonschema +// repository-desc An implementation of JSON Schema, based on IETF's draft v4 - Go language. +// +// description Contains const string and messages. +// +// created 01-01-2015 + +package gojsonschema + +type ( + // locale is an interface for defining custom error strings + locale interface { + Required() string + InvalidType() string + NumberAnyOf() string + NumberOneOf() string + NumberAllOf() string + NumberNot() string + MissingDependency() string + Internal() string + Const() string + Enum() string + ArrayNotEnoughItems() string + ArrayNoAdditionalItems() string + ArrayMinItems() string + ArrayMaxItems() string + Unique() string + ArrayContains() string + ArrayMinProperties() string + ArrayMaxProperties() string + AdditionalPropertyNotAllowed() string + InvalidPropertyPattern() string + InvalidPropertyName() string + StringGTE() string + StringLTE() string + DoesNotMatchPattern() string + DoesNotMatchFormat() string + MultipleOf() string + NumberGTE() string + NumberGT() string + NumberLTE() string + NumberLT() string + + // Schema validations + RegexPattern() string + GreaterThanZero() string + MustBeOfA() string + MustBeOfAn() string + CannotBeUsedWithout() string + CannotBeGT() string + MustBeOfType() string + MustBeValidRegex() string + MustBeValidFormat() string + MustBeGTEZero() string + KeyCannotBeGreaterThan() string + KeyItemsMustBeOfType() string + KeyItemsMustBeUnique() string + ReferenceMustBeCanonical() string + NotAValidType() string + Duplicated() string + HttpBadStatus() string + ParseError() string + + ConditionThen() string + ConditionElse() string + + // ErrorFormat + ErrorFormat() string + } + + // DefaultLocale is the default locale for this package + DefaultLocale struct{} +) + +func (l DefaultLocale) Required() string { + return `{{.property}} is required` +} + +func (l DefaultLocale) InvalidType() string { + return `Invalid type. Expected: {{.expected}}, given: {{.given}}` +} + +func (l DefaultLocale) NumberAnyOf() string { + return `Must validate at least one schema (anyOf)` +} + +func (l DefaultLocale) NumberOneOf() string { + return `Must validate one and only one schema (oneOf)` +} + +func (l DefaultLocale) NumberAllOf() string { + return `Must validate all the schemas (allOf)` +} + +func (l DefaultLocale) NumberNot() string { + return `Must not validate the schema (not)` +} + +func (l DefaultLocale) MissingDependency() string { + return `Has a dependency on {{.dependency}}` +} + +func (l DefaultLocale) Internal() string { + return `Internal Error {{.error}}` +} + +func (l DefaultLocale) Const() string { + return `{{.field}} does not match: {{.allowed}}` +} + +func (l DefaultLocale) Enum() string { + return `{{.field}} must be one of the following: {{.allowed}}` +} + +func (l DefaultLocale) ArrayNoAdditionalItems() string { + return `No additional items allowed on array` +} + +func (l DefaultLocale) ArrayNotEnoughItems() string { + return `Not enough items on array to match positional list of schema` +} + +func (l DefaultLocale) ArrayMinItems() string { + return `Array must have at least {{.min}} items` +} + +func (l DefaultLocale) ArrayMaxItems() string { + return `Array must have at most {{.max}} items` +} + +func (l DefaultLocale) Unique() string { + return `{{.type}} items must be unique` +} + +func (l DefaultLocale) ArrayContains() string { + return `At least one of the items must match` +} + +func (l DefaultLocale) ArrayMinProperties() string { + return `Must have at least {{.min}} properties` +} + +func (l DefaultLocale) ArrayMaxProperties() string { + return `Must have at most {{.max}} properties` +} + +func (l DefaultLocale) AdditionalPropertyNotAllowed() string { + return `Additional property {{.property}} is not allowed` +} + +func (l DefaultLocale) InvalidPropertyPattern() string { + return `Property "{{.property}}" does not match pattern {{.pattern}}` +} + +func (l DefaultLocale) InvalidPropertyName() string { + return `Property name of "{{.property}}" does not match` +} + +func (l DefaultLocale) StringGTE() string { + return `String length must be greater than or equal to {{.min}}` +} + +func (l DefaultLocale) StringLTE() string { + return `String length must be less than or equal to {{.max}}` +} + +func (l DefaultLocale) DoesNotMatchPattern() string { + return `Does not match pattern '{{.pattern}}'` +} + +func (l DefaultLocale) DoesNotMatchFormat() string { + return `Does not match format '{{.format}}'` +} + +func (l DefaultLocale) MultipleOf() string { + return `Must be a multiple of {{.multiple}}` +} + +func (l DefaultLocale) NumberGTE() string { + return `Must be greater than or equal to {{.min}}` +} + +func (l DefaultLocale) NumberGT() string { + return `Must be greater than {{.min}}` +} + +func (l DefaultLocale) NumberLTE() string { + return `Must be less than or equal to {{.max}}` +} + +func (l DefaultLocale) NumberLT() string { + return `Must be less than {{.max}}` +} + +// Schema validators +func (l DefaultLocale) RegexPattern() string { + return `Invalid regex pattern '{{.pattern}}'` +} + +func (l DefaultLocale) GreaterThanZero() string { + return `{{.number}} must be strictly greater than 0` +} + +func (l DefaultLocale) MustBeOfA() string { + return `{{.x}} must be of a {{.y}}` +} + +func (l DefaultLocale) MustBeOfAn() string { + return `{{.x}} must be of an {{.y}}` +} + +func (l DefaultLocale) CannotBeUsedWithout() string { + return `{{.x}} cannot be used without {{.y}}` +} + +func (l DefaultLocale) CannotBeGT() string { + return `{{.x}} cannot be greater than {{.y}}` +} + +func (l DefaultLocale) MustBeOfType() string { + return `{{.key}} must be of type {{.type}}` +} + +func (l DefaultLocale) MustBeValidRegex() string { + return `{{.key}} must be a valid regex` +} + +func (l DefaultLocale) MustBeValidFormat() string { + return `{{.key}} must be a valid format {{.given}}` +} + +func (l DefaultLocale) MustBeGTEZero() string { + return `{{.key}} must be greater than or equal to 0` +} + +func (l DefaultLocale) KeyCannotBeGreaterThan() string { + return `{{.key}} cannot be greater than {{.y}}` +} + +func (l DefaultLocale) KeyItemsMustBeOfType() string { + return `{{.key}} items must be {{.type}}` +} + +func (l DefaultLocale) KeyItemsMustBeUnique() string { + return `{{.key}} items must be unique` +} + +func (l DefaultLocale) ReferenceMustBeCanonical() string { + return `Reference {{.reference}} must be canonical` +} + +func (l DefaultLocale) NotAValidType() string { + return `has a primitive type that is NOT VALID -- given: {{.given}} Expected valid values are:{{.expected}}` +} + +func (l DefaultLocale) Duplicated() string { + return `{{.type}} type is duplicated` +} + +func (l DefaultLocale) HttpBadStatus() string { + return `Could not read schema from HTTP, response status is {{.status}}` +} + +// Replacement options: field, description, context, value +func (l DefaultLocale) ErrorFormat() string { + return `{{.field}}: {{.description}}` +} + +//Parse error +func (l DefaultLocale) ParseError() string { + return `Expected: {{.expected}}, given: Invalid JSON` +} + +//If/Else +func (l DefaultLocale) ConditionThen() string { + return `Must validate "then" as "if" was valid` +} + +func (l DefaultLocale) ConditionElse() string { + return `Must validate "else" as "if" was not valid` +} + +const ( + STRING_NUMBER = "number" + STRING_ARRAY_OF_STRINGS = "array of strings" + STRING_ARRAY_OF_SCHEMAS = "array of schemas" + STRING_SCHEMA = "valid schema" + STRING_SCHEMA_OR_ARRAY_OF_STRINGS = "schema or array of strings" + STRING_PROPERTIES = "properties" + STRING_DEPENDENCY = "dependency" + STRING_PROPERTY = "property" + STRING_UNDEFINED = "undefined" + STRING_CONTEXT_ROOT = "(root)" + STRING_ROOT_SCHEMA_PROPERTY = "(root)" +) diff --git a/tools/voluspa/vendor/github.com/xeipuuv/gojsonschema/result.go b/tools/voluspa/vendor/github.com/xeipuuv/gojsonschema/result.go new file mode 100644 index 00000000000..7896f312990 --- /dev/null +++ b/tools/voluspa/vendor/github.com/xeipuuv/gojsonschema/result.go @@ -0,0 +1,195 @@ +// Copyright 2015 xeipuuv ( https://github.com/xeipuuv ) +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// author xeipuuv +// author-github https://github.com/xeipuuv +// author-mail xeipuuv@gmail.com +// +// repository-name gojsonschema +// repository-desc An implementation of JSON Schema, based on IETF's draft v4 - Go language. +// +// description Result and ResultError implementations. +// +// created 01-01-2015 + +package gojsonschema + +import ( + "fmt" + "strings" +) + +type ( + // ErrorDetails is a map of details specific to each error. + // While the values will vary, every error will contain a "field" value + ErrorDetails map[string]interface{} + + // ResultError is the interface that library errors must implement + ResultError interface { + Field() string + SetType(string) + Type() string + SetContext(*JsonContext) + Context() *JsonContext + SetDescription(string) + Description() string + SetDescriptionFormat(string) + DescriptionFormat() string + SetValue(interface{}) + Value() interface{} + SetDetails(ErrorDetails) + Details() ErrorDetails + String() string + } + + // ResultErrorFields holds the fields for each ResultError implementation. + // ResultErrorFields implements the ResultError interface, so custom errors + // can be defined by just embedding this type + ResultErrorFields struct { + errorType string // A string with the type of error (i.e. invalid_type) + context *JsonContext // Tree like notation of the part that failed the validation. ex (root).a.b ... + description string // A human readable error message + descriptionFormat string // A format for human readable error message + value interface{} // Value given by the JSON file that is the source of the error + details ErrorDetails + } + + Result struct { + errors []ResultError + // Scores how well the validation matched. Useful in generating + // better error messages for anyOf and oneOf. + score int + } +) + +// Field outputs the field name without the root context +// i.e. firstName or person.firstName instead of (root).firstName or (root).person.firstName +func (v *ResultErrorFields) Field() string { + if p, ok := v.Details()["property"]; ok { + if str, isString := p.(string); isString { + return str + } + } + + return strings.TrimPrefix(v.context.String(), STRING_ROOT_SCHEMA_PROPERTY+".") +} + +func (v *ResultErrorFields) SetType(errorType string) { + v.errorType = errorType +} + +func (v *ResultErrorFields) Type() string { + return v.errorType +} + +func (v *ResultErrorFields) SetContext(context *JsonContext) { + v.context = context +} + +func (v *ResultErrorFields) Context() *JsonContext { + return v.context +} + +func (v *ResultErrorFields) SetDescription(description string) { + v.description = description +} + +func (v *ResultErrorFields) Description() string { + return v.description +} + +func (v *ResultErrorFields) SetDescriptionFormat(descriptionFormat string) { + v.descriptionFormat = descriptionFormat +} + +func (v *ResultErrorFields) DescriptionFormat() string { + return v.descriptionFormat +} + +func (v *ResultErrorFields) SetValue(value interface{}) { + v.value = value +} + +func (v *ResultErrorFields) Value() interface{} { + return v.value +} + +func (v *ResultErrorFields) SetDetails(details ErrorDetails) { + v.details = details +} + +func (v *ResultErrorFields) Details() ErrorDetails { + return v.details +} + +func (v ResultErrorFields) String() string { + // as a fallback, the value is displayed go style + valueString := fmt.Sprintf("%v", v.value) + + // marshal the go value value to json + if v.value == nil { + valueString = TYPE_NULL + } else { + if vs, err := marshalToJsonString(v.value); err == nil { + if vs == nil { + valueString = TYPE_NULL + } else { + valueString = *vs + } + } + } + + return formatErrorDescription(Locale.ErrorFormat(), ErrorDetails{ + "context": v.context.String(), + "description": v.description, + "value": valueString, + "field": v.Field(), + }) +} + +func (v *Result) Valid() bool { + return len(v.errors) == 0 +} + +func (v *Result) Errors() []ResultError { + return v.errors +} + +// Add a fully filled error to the error set +// SetDescription() will be called with the result of the parsed err.DescriptionFormat() +func (v *Result) AddError(err ResultError, details ErrorDetails) { + if _, exists := details["context"]; !exists && err.Context() != nil { + details["context"] = err.Context().String() + } + + err.SetDescription(formatErrorDescription(err.DescriptionFormat(), details)) + + v.errors = append(v.errors, err) +} + +func (v *Result) addInternalError(err ResultError, context *JsonContext, value interface{}, details ErrorDetails) { + newError(err, context, value, Locale, details) + v.errors = append(v.errors, err) + v.score -= 2 // results in a net -1 when added to the +1 we get at the end of the validation function +} + +// Used to copy errors from a sub-schema to the main one +func (v *Result) mergeErrors(otherResult *Result) { + v.errors = append(v.errors, otherResult.Errors()...) + v.score += otherResult.score +} + +func (v *Result) incrementScore() { + v.score++ +} diff --git a/tools/voluspa/vendor/github.com/xeipuuv/gojsonschema/schema.go b/tools/voluspa/vendor/github.com/xeipuuv/gojsonschema/schema.go new file mode 100644 index 00000000000..4ae3c62047a --- /dev/null +++ b/tools/voluspa/vendor/github.com/xeipuuv/gojsonschema/schema.go @@ -0,0 +1,1000 @@ +// Copyright 2015 xeipuuv ( https://github.com/xeipuuv ) +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// author xeipuuv +// author-github https://github.com/xeipuuv +// author-mail xeipuuv@gmail.com +// +// repository-name gojsonschema +// repository-desc An implementation of JSON Schema, based on IETF's draft v4 - Go language. +// +// description Defines Schema, the main entry to every subSchema. +// Contains the parsing logic and error checking. +// +// created 26-02-2013 + +package gojsonschema + +import ( + "errors" + "math/big" + "reflect" + "regexp" + "text/template" + + "github.com/xeipuuv/gojsonreference" +) + +var ( + // Locale is the default locale to use + // Library users can overwrite with their own implementation + Locale locale = DefaultLocale{} + + // ErrorTemplateFuncs allows you to define custom template funcs for use in localization. + ErrorTemplateFuncs template.FuncMap +) + +func NewSchema(l JSONLoader) (*Schema, error) { + return NewSchemaLoader().Compile(l) +} + +type Schema struct { + documentReference gojsonreference.JsonReference + rootSchema *subSchema + pool *schemaPool + referencePool *schemaReferencePool +} + +func (d *Schema) parse(document interface{}) error { + d.rootSchema = &subSchema{property: STRING_ROOT_SCHEMA_PROPERTY} + return d.parseSchema(document, d.rootSchema) +} + +func (d *Schema) SetRootSchemaName(name string) { + d.rootSchema.property = name +} + +// Parses a subSchema +// +// Pretty long function ( sorry :) )... but pretty straight forward, repetitive and boring +// Not much magic involved here, most of the job is to validate the key names and their values, +// then the values are copied into subSchema struct +// +func (d *Schema) parseSchema(documentNode interface{}, currentSchema *subSchema) error { + + // As of draft 6 "true" is equivalent to an empty schema "{}" and false equals "{"not":{}}" + if isKind(documentNode, reflect.Bool) { + b := documentNode.(bool) + if b { + documentNode = map[string]interface{}{} + } else { + documentNode = map[string]interface{}{"not": true} + } + } + + if !isKind(documentNode, reflect.Map) { + return errors.New(formatErrorDescription( + Locale.ParseError(), + ErrorDetails{ + "expected": STRING_SCHEMA, + }, + )) + } + + m := documentNode.(map[string]interface{}) + + if currentSchema.parent == nil { + currentSchema.ref = &d.documentReference + currentSchema.id = &d.documentReference + + if existsMapKey(m, KEY_SCHEMA) && false { + if !isKind(m[KEY_SCHEMA], reflect.String) { + return errors.New(formatErrorDescription( + Locale.InvalidType(), + ErrorDetails{ + "expected": TYPE_STRING, + "given": KEY_SCHEMA, + }, + )) + } + } + } + + if currentSchema.id == nil && currentSchema.parent != nil { + currentSchema.id = currentSchema.parent.id + } + + // In draft 6 the id keyword was renamed to $id + // Use the old id by default + keyID := KEY_ID_NEW + if existsMapKey(m, KEY_ID) { + keyID = KEY_ID + } + if existsMapKey(m, keyID) && !isKind(m[keyID], reflect.String) { + return errors.New(formatErrorDescription( + Locale.InvalidType(), + ErrorDetails{ + "expected": TYPE_STRING, + "given": keyID, + }, + )) + } + if k, ok := m[keyID].(string); ok { + jsonReference, err := gojsonreference.NewJsonReference(k) + if err != nil { + return err + } + if currentSchema == d.rootSchema { + currentSchema.id = &jsonReference + } else { + ref, err := currentSchema.parent.id.Inherits(jsonReference) + if err != nil { + return err + } + currentSchema.id = ref + } + } + + // definitions + if existsMapKey(m, KEY_DEFINITIONS) { + if isKind(m[KEY_DEFINITIONS], reflect.Map, reflect.Bool) { + for _, dv := range m[KEY_DEFINITIONS].(map[string]interface{}) { + if isKind(dv, reflect.Map, reflect.Bool) { + + newSchema := &subSchema{property: KEY_DEFINITIONS, parent: currentSchema} + + err := d.parseSchema(dv, newSchema) + + if err != nil { + return err + } + } else { + return errors.New(formatErrorDescription( + Locale.InvalidType(), + ErrorDetails{ + "expected": STRING_ARRAY_OF_SCHEMAS, + "given": KEY_DEFINITIONS, + }, + )) + } + } + } else { + return errors.New(formatErrorDescription( + Locale.InvalidType(), + ErrorDetails{ + "expected": STRING_ARRAY_OF_SCHEMAS, + "given": KEY_DEFINITIONS, + }, + )) + } + + } + + // title + if existsMapKey(m, KEY_TITLE) && !isKind(m[KEY_TITLE], reflect.String) { + return errors.New(formatErrorDescription( + Locale.InvalidType(), + ErrorDetails{ + "expected": TYPE_STRING, + "given": KEY_TITLE, + }, + )) + } + if k, ok := m[KEY_TITLE].(string); ok { + currentSchema.title = &k + } + + // description + if existsMapKey(m, KEY_DESCRIPTION) && !isKind(m[KEY_DESCRIPTION], reflect.String) { + return errors.New(formatErrorDescription( + Locale.InvalidType(), + ErrorDetails{ + "expected": TYPE_STRING, + "given": KEY_DESCRIPTION, + }, + )) + } + if k, ok := m[KEY_DESCRIPTION].(string); ok { + currentSchema.description = &k + } + + // $ref + if existsMapKey(m, KEY_REF) && !isKind(m[KEY_REF], reflect.String) { + return errors.New(formatErrorDescription( + Locale.InvalidType(), + ErrorDetails{ + "expected": TYPE_STRING, + "given": KEY_REF, + }, + )) + } + + if k, ok := m[KEY_REF].(string); ok { + + jsonReference, err := gojsonreference.NewJsonReference(k) + if err != nil { + return err + } + + currentSchema.ref = &jsonReference + + if sch, ok := d.referencePool.Get(currentSchema.ref.String()); ok { + currentSchema.refSchema = sch + } else { + err := d.parseReference(documentNode, currentSchema) + + if err != nil { + return err + } + + return nil + } + } + + // type + if existsMapKey(m, KEY_TYPE) { + if isKind(m[KEY_TYPE], reflect.String) { + if k, ok := m[KEY_TYPE].(string); ok { + err := currentSchema.types.Add(k) + if err != nil { + return err + } + } + } else { + if isKind(m[KEY_TYPE], reflect.Slice) { + arrayOfTypes := m[KEY_TYPE].([]interface{}) + for _, typeInArray := range arrayOfTypes { + if reflect.ValueOf(typeInArray).Kind() != reflect.String { + return errors.New(formatErrorDescription( + Locale.InvalidType(), + ErrorDetails{ + "expected": TYPE_STRING + "/" + STRING_ARRAY_OF_STRINGS, + "given": KEY_TYPE, + }, + )) + } else { + currentSchema.types.Add(typeInArray.(string)) + } + } + + } else { + return errors.New(formatErrorDescription( + Locale.InvalidType(), + ErrorDetails{ + "expected": TYPE_STRING + "/" + STRING_ARRAY_OF_STRINGS, + "given": KEY_TYPE, + }, + )) + } + } + } + + // properties + if existsMapKey(m, KEY_PROPERTIES) { + err := d.parseProperties(m[KEY_PROPERTIES], currentSchema) + if err != nil { + return err + } + } + + // additionalProperties + if existsMapKey(m, KEY_ADDITIONAL_PROPERTIES) { + if isKind(m[KEY_ADDITIONAL_PROPERTIES], reflect.Bool) { + currentSchema.additionalProperties = m[KEY_ADDITIONAL_PROPERTIES].(bool) + } else if isKind(m[KEY_ADDITIONAL_PROPERTIES], reflect.Map) { + newSchema := &subSchema{property: KEY_ADDITIONAL_PROPERTIES, parent: currentSchema, ref: currentSchema.ref} + currentSchema.additionalProperties = newSchema + err := d.parseSchema(m[KEY_ADDITIONAL_PROPERTIES], newSchema) + if err != nil { + return errors.New(err.Error()) + } + } else { + return errors.New(formatErrorDescription( + Locale.InvalidType(), + ErrorDetails{ + "expected": TYPE_BOOLEAN + "/" + STRING_SCHEMA, + "given": KEY_ADDITIONAL_PROPERTIES, + }, + )) + } + } + + // patternProperties + if existsMapKey(m, KEY_PATTERN_PROPERTIES) { + if isKind(m[KEY_PATTERN_PROPERTIES], reflect.Map) { + patternPropertiesMap := m[KEY_PATTERN_PROPERTIES].(map[string]interface{}) + if len(patternPropertiesMap) > 0 { + currentSchema.patternProperties = make(map[string]*subSchema) + for k, v := range patternPropertiesMap { + _, err := regexp.MatchString(k, "") + if err != nil { + return errors.New(formatErrorDescription( + Locale.RegexPattern(), + ErrorDetails{"pattern": k}, + )) + } + newSchema := &subSchema{property: k, parent: currentSchema, ref: currentSchema.ref} + err = d.parseSchema(v, newSchema) + if err != nil { + return errors.New(err.Error()) + } + currentSchema.patternProperties[k] = newSchema + } + } + } else { + return errors.New(formatErrorDescription( + Locale.InvalidType(), + ErrorDetails{ + "expected": STRING_SCHEMA, + "given": KEY_PATTERN_PROPERTIES, + }, + )) + } + } + + // propertyNames + if existsMapKey(m, KEY_PROPERTY_NAMES) { + if isKind(m[KEY_PROPERTY_NAMES], reflect.Map, reflect.Bool) { + newSchema := &subSchema{property: KEY_PROPERTY_NAMES, parent: currentSchema, ref: currentSchema.ref} + currentSchema.propertyNames = newSchema + err := d.parseSchema(m[KEY_PROPERTY_NAMES], newSchema) + if err != nil { + return err + } + } else { + return errors.New(formatErrorDescription( + Locale.InvalidType(), + ErrorDetails{ + "expected": STRING_SCHEMA, + "given": KEY_PATTERN_PROPERTIES, + }, + )) + } + } + + // dependencies + if existsMapKey(m, KEY_DEPENDENCIES) { + err := d.parseDependencies(m[KEY_DEPENDENCIES], currentSchema) + if err != nil { + return err + } + } + + // items + if existsMapKey(m, KEY_ITEMS) { + if isKind(m[KEY_ITEMS], reflect.Slice) { + for _, itemElement := range m[KEY_ITEMS].([]interface{}) { + if isKind(itemElement, reflect.Map, reflect.Bool) { + newSchema := &subSchema{parent: currentSchema, property: KEY_ITEMS} + newSchema.ref = currentSchema.ref + currentSchema.AddItemsChild(newSchema) + err := d.parseSchema(itemElement, newSchema) + if err != nil { + return err + } + } else { + return errors.New(formatErrorDescription( + Locale.InvalidType(), + ErrorDetails{ + "expected": STRING_SCHEMA + "/" + STRING_ARRAY_OF_SCHEMAS, + "given": KEY_ITEMS, + }, + )) + } + currentSchema.itemsChildrenIsSingleSchema = false + } + } else if isKind(m[KEY_ITEMS], reflect.Map, reflect.Bool) { + newSchema := &subSchema{parent: currentSchema, property: KEY_ITEMS} + newSchema.ref = currentSchema.ref + currentSchema.AddItemsChild(newSchema) + err := d.parseSchema(m[KEY_ITEMS], newSchema) + if err != nil { + return err + } + currentSchema.itemsChildrenIsSingleSchema = true + } else { + return errors.New(formatErrorDescription( + Locale.InvalidType(), + ErrorDetails{ + "expected": STRING_SCHEMA + "/" + STRING_ARRAY_OF_SCHEMAS, + "given": KEY_ITEMS, + }, + )) + } + } + + // additionalItems + if existsMapKey(m, KEY_ADDITIONAL_ITEMS) { + if isKind(m[KEY_ADDITIONAL_ITEMS], reflect.Bool) { + currentSchema.additionalItems = m[KEY_ADDITIONAL_ITEMS].(bool) + } else if isKind(m[KEY_ADDITIONAL_ITEMS], reflect.Map) { + newSchema := &subSchema{property: KEY_ADDITIONAL_ITEMS, parent: currentSchema, ref: currentSchema.ref} + currentSchema.additionalItems = newSchema + err := d.parseSchema(m[KEY_ADDITIONAL_ITEMS], newSchema) + if err != nil { + return errors.New(err.Error()) + } + } else { + return errors.New(formatErrorDescription( + Locale.InvalidType(), + ErrorDetails{ + "expected": TYPE_BOOLEAN + "/" + STRING_SCHEMA, + "given": KEY_ADDITIONAL_ITEMS, + }, + )) + } + } + + // validation : number / integer + + if existsMapKey(m, KEY_MULTIPLE_OF) { + multipleOfValue := mustBeNumber(m[KEY_MULTIPLE_OF]) + if multipleOfValue == nil { + return errors.New(formatErrorDescription( + Locale.InvalidType(), + ErrorDetails{ + "expected": STRING_NUMBER, + "given": KEY_MULTIPLE_OF, + }, + )) + } + if multipleOfValue.Cmp(big.NewFloat(0)) <= 0 { + return errors.New(formatErrorDescription( + Locale.GreaterThanZero(), + ErrorDetails{"number": KEY_MULTIPLE_OF}, + )) + } + currentSchema.multipleOf = multipleOfValue + } + + if existsMapKey(m, KEY_MINIMUM) { + minimumValue := mustBeNumber(m[KEY_MINIMUM]) + if minimumValue == nil { + return errors.New(formatErrorDescription( + Locale.MustBeOfA(), + ErrorDetails{"x": KEY_MINIMUM, "y": STRING_NUMBER}, + )) + } + currentSchema.minimum = minimumValue + } + + if existsMapKey(m, KEY_EXCLUSIVE_MINIMUM) { + if isKind(m[KEY_EXCLUSIVE_MINIMUM], reflect.Bool) { + if currentSchema.minimum == nil { + return errors.New(formatErrorDescription( + Locale.CannotBeUsedWithout(), + ErrorDetails{"x": KEY_EXCLUSIVE_MINIMUM, "y": KEY_MINIMUM}, + )) + } + exclusiveMinimumValue := m[KEY_EXCLUSIVE_MINIMUM].(bool) + currentSchema.exclusiveMinimum = exclusiveMinimumValue + } else if isJsonNumber(m[KEY_EXCLUSIVE_MINIMUM]) { + minimumValue := mustBeNumber(m[KEY_EXCLUSIVE_MINIMUM]) + + currentSchema.minimum = minimumValue + currentSchema.exclusiveMinimum = true + } else { + return errors.New(formatErrorDescription( + Locale.InvalidType(), + ErrorDetails{"expected": TYPE_BOOLEAN + ", " + TYPE_NUMBER, "given": KEY_EXCLUSIVE_MINIMUM}, + )) + } + } + + if existsMapKey(m, KEY_MAXIMUM) { + maximumValue := mustBeNumber(m[KEY_MAXIMUM]) + if maximumValue == nil { + return errors.New(formatErrorDescription( + Locale.MustBeOfA(), + ErrorDetails{"x": KEY_MAXIMUM, "y": STRING_NUMBER}, + )) + } + currentSchema.maximum = maximumValue + } + + if existsMapKey(m, KEY_EXCLUSIVE_MAXIMUM) { + if isKind(m[KEY_EXCLUSIVE_MAXIMUM], reflect.Bool) { + if currentSchema.maximum == nil { + return errors.New(formatErrorDescription( + Locale.CannotBeUsedWithout(), + ErrorDetails{"x": KEY_EXCLUSIVE_MAXIMUM, "y": KEY_MAXIMUM}, + )) + } + exclusiveMaximumValue := m[KEY_EXCLUSIVE_MAXIMUM].(bool) + currentSchema.exclusiveMaximum = exclusiveMaximumValue + } else if isJsonNumber(m[KEY_EXCLUSIVE_MAXIMUM]) { + maximumValue := mustBeNumber(m[KEY_EXCLUSIVE_MAXIMUM]) + + currentSchema.maximum = maximumValue + currentSchema.exclusiveMaximum = true + } else { + return errors.New(formatErrorDescription( + Locale.InvalidType(), + ErrorDetails{"expected": TYPE_BOOLEAN + ", " + TYPE_NUMBER, "given": KEY_EXCLUSIVE_MAXIMUM}, + )) + } + } + + if currentSchema.minimum != nil && currentSchema.maximum != nil { + if currentSchema.minimum.Cmp(currentSchema.maximum) == 1 { + return errors.New(formatErrorDescription( + Locale.CannotBeGT(), + ErrorDetails{"x": KEY_MINIMUM, "y": KEY_MAXIMUM}, + )) + } + } + + // validation : string + + if existsMapKey(m, KEY_MIN_LENGTH) { + minLengthIntegerValue := mustBeInteger(m[KEY_MIN_LENGTH]) + if minLengthIntegerValue == nil { + return errors.New(formatErrorDescription( + Locale.MustBeOfAn(), + ErrorDetails{"x": KEY_MIN_LENGTH, "y": TYPE_INTEGER}, + )) + } + if *minLengthIntegerValue < 0 { + return errors.New(formatErrorDescription( + Locale.MustBeGTEZero(), + ErrorDetails{"key": KEY_MIN_LENGTH}, + )) + } + currentSchema.minLength = minLengthIntegerValue + } + + if existsMapKey(m, KEY_MAX_LENGTH) { + maxLengthIntegerValue := mustBeInteger(m[KEY_MAX_LENGTH]) + if maxLengthIntegerValue == nil { + return errors.New(formatErrorDescription( + Locale.MustBeOfAn(), + ErrorDetails{"x": KEY_MAX_LENGTH, "y": TYPE_INTEGER}, + )) + } + if *maxLengthIntegerValue < 0 { + return errors.New(formatErrorDescription( + Locale.MustBeGTEZero(), + ErrorDetails{"key": KEY_MAX_LENGTH}, + )) + } + currentSchema.maxLength = maxLengthIntegerValue + } + + if currentSchema.minLength != nil && currentSchema.maxLength != nil { + if *currentSchema.minLength > *currentSchema.maxLength { + return errors.New(formatErrorDescription( + Locale.CannotBeGT(), + ErrorDetails{"x": KEY_MIN_LENGTH, "y": KEY_MAX_LENGTH}, + )) + } + } + + if existsMapKey(m, KEY_PATTERN) { + if isKind(m[KEY_PATTERN], reflect.String) { + regexpObject, err := regexp.Compile(m[KEY_PATTERN].(string)) + if err != nil { + return errors.New(formatErrorDescription( + Locale.MustBeValidRegex(), + ErrorDetails{"key": KEY_PATTERN}, + )) + } + currentSchema.pattern = regexpObject + } else { + return errors.New(formatErrorDescription( + Locale.MustBeOfA(), + ErrorDetails{"x": KEY_PATTERN, "y": TYPE_STRING}, + )) + } + } + + if existsMapKey(m, KEY_FORMAT) { + formatString, ok := m[KEY_FORMAT].(string) + if ok && FormatCheckers.Has(formatString) { + currentSchema.format = formatString + } + } + + // validation : object + + if existsMapKey(m, KEY_MIN_PROPERTIES) { + minPropertiesIntegerValue := mustBeInteger(m[KEY_MIN_PROPERTIES]) + if minPropertiesIntegerValue == nil { + return errors.New(formatErrorDescription( + Locale.MustBeOfAn(), + ErrorDetails{"x": KEY_MIN_PROPERTIES, "y": TYPE_INTEGER}, + )) + } + if *minPropertiesIntegerValue < 0 { + return errors.New(formatErrorDescription( + Locale.MustBeGTEZero(), + ErrorDetails{"key": KEY_MIN_PROPERTIES}, + )) + } + currentSchema.minProperties = minPropertiesIntegerValue + } + + if existsMapKey(m, KEY_MAX_PROPERTIES) { + maxPropertiesIntegerValue := mustBeInteger(m[KEY_MAX_PROPERTIES]) + if maxPropertiesIntegerValue == nil { + return errors.New(formatErrorDescription( + Locale.MustBeOfAn(), + ErrorDetails{"x": KEY_MAX_PROPERTIES, "y": TYPE_INTEGER}, + )) + } + if *maxPropertiesIntegerValue < 0 { + return errors.New(formatErrorDescription( + Locale.MustBeGTEZero(), + ErrorDetails{"key": KEY_MAX_PROPERTIES}, + )) + } + currentSchema.maxProperties = maxPropertiesIntegerValue + } + + if currentSchema.minProperties != nil && currentSchema.maxProperties != nil { + if *currentSchema.minProperties > *currentSchema.maxProperties { + return errors.New(formatErrorDescription( + Locale.KeyCannotBeGreaterThan(), + ErrorDetails{"key": KEY_MIN_PROPERTIES, "y": KEY_MAX_PROPERTIES}, + )) + } + } + + if existsMapKey(m, KEY_REQUIRED) { + if isKind(m[KEY_REQUIRED], reflect.Slice) { + requiredValues := m[KEY_REQUIRED].([]interface{}) + for _, requiredValue := range requiredValues { + if isKind(requiredValue, reflect.String) { + err := currentSchema.AddRequired(requiredValue.(string)) + if err != nil { + return err + } + } else { + return errors.New(formatErrorDescription( + Locale.KeyItemsMustBeOfType(), + ErrorDetails{"key": KEY_REQUIRED, "type": TYPE_STRING}, + )) + } + } + } else { + return errors.New(formatErrorDescription( + Locale.MustBeOfAn(), + ErrorDetails{"x": KEY_REQUIRED, "y": TYPE_ARRAY}, + )) + } + } + + // validation : array + + if existsMapKey(m, KEY_MIN_ITEMS) { + minItemsIntegerValue := mustBeInteger(m[KEY_MIN_ITEMS]) + if minItemsIntegerValue == nil { + return errors.New(formatErrorDescription( + Locale.MustBeOfAn(), + ErrorDetails{"x": KEY_MIN_ITEMS, "y": TYPE_INTEGER}, + )) + } + if *minItemsIntegerValue < 0 { + return errors.New(formatErrorDescription( + Locale.MustBeGTEZero(), + ErrorDetails{"key": KEY_MIN_ITEMS}, + )) + } + currentSchema.minItems = minItemsIntegerValue + } + + if existsMapKey(m, KEY_MAX_ITEMS) { + maxItemsIntegerValue := mustBeInteger(m[KEY_MAX_ITEMS]) + if maxItemsIntegerValue == nil { + return errors.New(formatErrorDescription( + Locale.MustBeOfAn(), + ErrorDetails{"x": KEY_MAX_ITEMS, "y": TYPE_INTEGER}, + )) + } + if *maxItemsIntegerValue < 0 { + return errors.New(formatErrorDescription( + Locale.MustBeGTEZero(), + ErrorDetails{"key": KEY_MAX_ITEMS}, + )) + } + currentSchema.maxItems = maxItemsIntegerValue + } + + if existsMapKey(m, KEY_UNIQUE_ITEMS) { + if isKind(m[KEY_UNIQUE_ITEMS], reflect.Bool) { + currentSchema.uniqueItems = m[KEY_UNIQUE_ITEMS].(bool) + } else { + return errors.New(formatErrorDescription( + Locale.MustBeOfA(), + ErrorDetails{"x": KEY_UNIQUE_ITEMS, "y": TYPE_BOOLEAN}, + )) + } + } + + if existsMapKey(m, KEY_CONTAINS) { + newSchema := &subSchema{property: KEY_CONTAINS, parent: currentSchema, ref: currentSchema.ref} + currentSchema.contains = newSchema + err := d.parseSchema(m[KEY_CONTAINS], newSchema) + if err != nil { + return err + } + } + + // validation : all + + if existsMapKey(m, KEY_CONST) { + err := currentSchema.AddConst(m[KEY_CONST]) + if err != nil { + return err + } + } + + if existsMapKey(m, KEY_ENUM) { + if isKind(m[KEY_ENUM], reflect.Slice) { + for _, v := range m[KEY_ENUM].([]interface{}) { + err := currentSchema.AddEnum(v) + if err != nil { + return err + } + } + } else { + return errors.New(formatErrorDescription( + Locale.MustBeOfAn(), + ErrorDetails{"x": KEY_ENUM, "y": TYPE_ARRAY}, + )) + } + } + + // validation : subSchema + + if existsMapKey(m, KEY_ONE_OF) { + if isKind(m[KEY_ONE_OF], reflect.Slice) { + for _, v := range m[KEY_ONE_OF].([]interface{}) { + newSchema := &subSchema{property: KEY_ONE_OF, parent: currentSchema, ref: currentSchema.ref} + currentSchema.AddOneOf(newSchema) + err := d.parseSchema(v, newSchema) + if err != nil { + return err + } + } + } else { + return errors.New(formatErrorDescription( + Locale.MustBeOfAn(), + ErrorDetails{"x": KEY_ONE_OF, "y": TYPE_ARRAY}, + )) + } + } + + if existsMapKey(m, KEY_ANY_OF) { + if isKind(m[KEY_ANY_OF], reflect.Slice) { + for _, v := range m[KEY_ANY_OF].([]interface{}) { + newSchema := &subSchema{property: KEY_ANY_OF, parent: currentSchema, ref: currentSchema.ref} + currentSchema.AddAnyOf(newSchema) + err := d.parseSchema(v, newSchema) + if err != nil { + return err + } + } + } else { + return errors.New(formatErrorDescription( + Locale.MustBeOfAn(), + ErrorDetails{"x": KEY_ANY_OF, "y": TYPE_ARRAY}, + )) + } + } + + if existsMapKey(m, KEY_ALL_OF) { + if isKind(m[KEY_ALL_OF], reflect.Slice) { + for _, v := range m[KEY_ALL_OF].([]interface{}) { + newSchema := &subSchema{property: KEY_ALL_OF, parent: currentSchema, ref: currentSchema.ref} + currentSchema.AddAllOf(newSchema) + err := d.parseSchema(v, newSchema) + if err != nil { + return err + } + } + } else { + return errors.New(formatErrorDescription( + Locale.MustBeOfAn(), + ErrorDetails{"x": KEY_ANY_OF, "y": TYPE_ARRAY}, + )) + } + } + + if existsMapKey(m, KEY_NOT) { + if isKind(m[KEY_NOT], reflect.Map, reflect.Bool) { + newSchema := &subSchema{property: KEY_NOT, parent: currentSchema, ref: currentSchema.ref} + currentSchema.SetNot(newSchema) + err := d.parseSchema(m[KEY_NOT], newSchema) + if err != nil { + return err + } + } else { + return errors.New(formatErrorDescription( + Locale.MustBeOfAn(), + ErrorDetails{"x": KEY_NOT, "y": TYPE_OBJECT}, + )) + } + } + + if existsMapKey(m, KEY_IF) { + if isKind(m[KEY_IF], reflect.Map, reflect.Bool) { + newSchema := &subSchema{property: KEY_IF, parent: currentSchema, ref: currentSchema.ref} + currentSchema.SetIf(newSchema) + err := d.parseSchema(m[KEY_IF], newSchema) + if err != nil { + return err + } + } else { + return errors.New(formatErrorDescription( + Locale.MustBeOfAn(), + ErrorDetails{"x": KEY_IF, "y": TYPE_OBJECT}, + )) + } + } + + if existsMapKey(m, KEY_THEN) { + if isKind(m[KEY_THEN], reflect.Map, reflect.Bool) { + newSchema := &subSchema{property: KEY_THEN, parent: currentSchema, ref: currentSchema.ref} + currentSchema.SetThen(newSchema) + err := d.parseSchema(m[KEY_THEN], newSchema) + if err != nil { + return err + } + } else { + return errors.New(formatErrorDescription( + Locale.MustBeOfAn(), + ErrorDetails{"x": KEY_THEN, "y": TYPE_OBJECT}, + )) + } + } + + if existsMapKey(m, KEY_ELSE) { + if isKind(m[KEY_ELSE], reflect.Map, reflect.Bool) { + newSchema := &subSchema{property: KEY_ELSE, parent: currentSchema, ref: currentSchema.ref} + currentSchema.SetElse(newSchema) + err := d.parseSchema(m[KEY_ELSE], newSchema) + if err != nil { + return err + } + } else { + return errors.New(formatErrorDescription( + Locale.MustBeOfAn(), + ErrorDetails{"x": KEY_ELSE, "y": TYPE_OBJECT}, + )) + } + } + + return nil +} + +func (d *Schema) parseReference(documentNode interface{}, currentSchema *subSchema) error { + var ( + refdDocumentNode interface{} + dsp *schemaPoolDocument + err error + ) + + newSchema := &subSchema{property: KEY_REF, parent: currentSchema, ref: currentSchema.ref} + + d.referencePool.Add(currentSchema.ref.String(), newSchema) + + dsp, err = d.pool.GetDocument(*currentSchema.ref) + if err != nil { + return err + } + newSchema.id = currentSchema.ref + + refdDocumentNode = dsp.Document + + if err != nil { + return err + } + + if !isKind(refdDocumentNode, reflect.Map, reflect.Bool) { + return errors.New(formatErrorDescription( + Locale.MustBeOfType(), + ErrorDetails{"key": STRING_SCHEMA, "type": TYPE_OBJECT}, + )) + } + + err = d.parseSchema(refdDocumentNode, newSchema) + if err != nil { + return err + } + + currentSchema.refSchema = newSchema + + return nil + +} + +func (d *Schema) parseProperties(documentNode interface{}, currentSchema *subSchema) error { + + if !isKind(documentNode, reflect.Map) { + return errors.New(formatErrorDescription( + Locale.MustBeOfType(), + ErrorDetails{"key": STRING_PROPERTIES, "type": TYPE_OBJECT}, + )) + } + + m := documentNode.(map[string]interface{}) + for k := range m { + schemaProperty := k + newSchema := &subSchema{property: schemaProperty, parent: currentSchema, ref: currentSchema.ref} + currentSchema.AddPropertiesChild(newSchema) + err := d.parseSchema(m[k], newSchema) + if err != nil { + return err + } + } + + return nil +} + +func (d *Schema) parseDependencies(documentNode interface{}, currentSchema *subSchema) error { + + if !isKind(documentNode, reflect.Map) { + return errors.New(formatErrorDescription( + Locale.MustBeOfType(), + ErrorDetails{"key": KEY_DEPENDENCIES, "type": TYPE_OBJECT}, + )) + } + + m := documentNode.(map[string]interface{}) + currentSchema.dependencies = make(map[string]interface{}) + + for k := range m { + switch reflect.ValueOf(m[k]).Kind() { + + case reflect.Slice: + values := m[k].([]interface{}) + var valuesToRegister []string + + for _, value := range values { + if !isKind(value, reflect.String) { + return errors.New(formatErrorDescription( + Locale.MustBeOfType(), + ErrorDetails{ + "key": STRING_DEPENDENCY, + "type": STRING_SCHEMA_OR_ARRAY_OF_STRINGS, + }, + )) + } else { + valuesToRegister = append(valuesToRegister, value.(string)) + } + currentSchema.dependencies[k] = valuesToRegister + } + + case reflect.Map, reflect.Bool: + depSchema := &subSchema{property: k, parent: currentSchema, ref: currentSchema.ref} + err := d.parseSchema(m[k], depSchema) + if err != nil { + return err + } + currentSchema.dependencies[k] = depSchema + + default: + return errors.New(formatErrorDescription( + Locale.MustBeOfType(), + ErrorDetails{ + "key": STRING_DEPENDENCY, + "type": STRING_SCHEMA_OR_ARRAY_OF_STRINGS, + }, + )) + } + + } + + return nil +} diff --git a/tools/voluspa/vendor/github.com/xeipuuv/gojsonschema/schemaLoader.go b/tools/voluspa/vendor/github.com/xeipuuv/gojsonschema/schemaLoader.go new file mode 100644 index 00000000000..09d3a37e255 --- /dev/null +++ b/tools/voluspa/vendor/github.com/xeipuuv/gojsonschema/schemaLoader.go @@ -0,0 +1,102 @@ +package gojsonschema + +import ( + "github.com/xeipuuv/gojsonreference" +) + +type SchemaLoader struct { + pool *schemaPool +} + +func NewSchemaLoader() *SchemaLoader { + + ps := &SchemaLoader{ + pool: &schemaPool{ + schemaPoolDocuments: make(map[string]*schemaPoolDocument), + }, + } + + return ps +} + +// AddSchemas adds an arbritrary amount of schemas to the schema cache. As this function does not require +// an explicit URL, every schema should contain an $id, so that it can be referenced by the main schema +func (sl *SchemaLoader) AddSchemas(loaders ...JSONLoader) error { + emptyRef, _ := gojsonreference.NewJsonReference("") + + for _, loader := range loaders { + doc, err := loader.LoadJSON() + if err != nil { + return err + } + // Directly use the Recursive function, so that it get only added to the schema pool by $id + // and not by the ref of the document as it's empty + if err = sl.pool.parseReferencesRecursive(doc, emptyRef); err != nil { + return err + } + } + + return nil +} + +//AddSchema adds a schema under the provided URL to the schema cache +func (sl *SchemaLoader) AddSchema(url string, loader JSONLoader) error { + + ref, err := gojsonreference.NewJsonReference(url) + + if err != nil { + return err + } + + doc, err := loader.LoadJSON() + + if err != nil { + return err + } + + return sl.pool.ParseReferences(doc, ref) +} + +func (sl *SchemaLoader) Compile(rootSchema JSONLoader) (*Schema, error) { + + ref, err := rootSchema.JsonReference() + + if err != nil { + return nil, err + } + + d := Schema{} + d.pool = sl.pool + d.pool.jsonLoaderFactory = rootSchema.LoaderFactory() + d.documentReference = ref + d.referencePool = newSchemaReferencePool() + + var doc interface{} + if ref.String() != "" { + // Get document from schema pool + spd, err := d.pool.GetDocument(d.documentReference) + if err != nil { + return nil, err + } + doc = spd.Document + } else { + // Load JSON directly + doc, err = rootSchema.LoadJSON() + if err != nil { + return nil, err + } + // References need only be parsed if loading JSON directly + // as pool.GetDocument already does this for us if loading by reference + err = d.pool.ParseReferences(doc, ref) + if err != nil { + return nil, err + } + } + + err = d.parse(doc) + if err != nil { + return nil, err + } + + return &d, nil +} diff --git a/tools/voluspa/vendor/github.com/xeipuuv/gojsonschema/schemaPool.go b/tools/voluspa/vendor/github.com/xeipuuv/gojsonschema/schemaPool.go new file mode 100644 index 00000000000..c3de8e5ef3f --- /dev/null +++ b/tools/voluspa/vendor/github.com/xeipuuv/gojsonschema/schemaPool.go @@ -0,0 +1,201 @@ +// Copyright 2015 xeipuuv ( https://github.com/xeipuuv ) +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// author xeipuuv +// author-github https://github.com/xeipuuv +// author-mail xeipuuv@gmail.com +// +// repository-name gojsonschema +// repository-desc An implementation of JSON Schema, based on IETF's draft v4 - Go language. +// +// description Defines resources pooling. +// Eases referencing and avoids downloading the same resource twice. +// +// created 26-02-2013 + +package gojsonschema + +import ( + "errors" + "fmt" + "reflect" + + "github.com/xeipuuv/gojsonreference" +) + +type schemaPoolDocument struct { + Document interface{} +} + +type schemaPool struct { + schemaPoolDocuments map[string]*schemaPoolDocument + jsonLoaderFactory JSONLoaderFactory +} + +func newSchemaPool(f JSONLoaderFactory) *schemaPool { + + p := &schemaPool{} + p.schemaPoolDocuments = make(map[string]*schemaPoolDocument) + p.jsonLoaderFactory = f + + return p +} + +func (p *schemaPool) ParseReferences(document interface{}, ref gojsonreference.JsonReference) error { + // Only the root document should be added to the schema pool + if _, ok := p.schemaPoolDocuments[ref.String()]; ok { + return fmt.Errorf("Reference already exists: \"%s\"", ref.String()) + } + err := p.parseReferencesRecursive(document, ref) + p.schemaPoolDocuments[ref.String()] = &schemaPoolDocument{Document: document} + return err +} + +func (p *schemaPool) parseReferencesRecursive(document interface{}, ref gojsonreference.JsonReference) error { + // parseReferencesRecursive parses a JSON document and resolves all $id and $ref references. + // For $ref references it takes into account the $id scope it is in and replaces + // the reference by the absolute resolved reference + + // When encountering errors it fails silently. Error handling is done when the schema + // is syntactically parsed and any error encountered here should also come up there. + switch m := document.(type) { + case []interface{}: + for _, v := range m { + p.parseReferencesRecursive(v, ref) + } + case map[string]interface{}: + localRef := &ref + + keyID := KEY_ID_NEW + if existsMapKey(m, KEY_ID) { + keyID = KEY_ID + } + if existsMapKey(m, keyID) && isKind(m[keyID], reflect.String) { + jsonReference, err := gojsonreference.NewJsonReference(m[keyID].(string)) + if err == nil { + localRef, err = ref.Inherits(jsonReference) + if err == nil { + if _, ok := p.schemaPoolDocuments[localRef.String()]; ok { + return fmt.Errorf("Reference already exists: \"%s\"", localRef.String()) + } + p.schemaPoolDocuments[localRef.String()] = &schemaPoolDocument{Document: document} + } + } + } + + if existsMapKey(m, KEY_REF) && isKind(m[KEY_REF], reflect.String) { + jsonReference, err := gojsonreference.NewJsonReference(m[KEY_REF].(string)) + if err == nil { + absoluteRef, err := localRef.Inherits(jsonReference) + if err == nil { + m[KEY_REF] = absoluteRef.String() + } + } + } + + for k, v := range m { + // const and enums should be interpreted literally, so ignore them + if k == KEY_CONST || k == KEY_ENUM { + continue + } + // Something like a property or a dependency is not a valid schema, as it might describe properties named "$ref", "$id" or "const", etc + // Therefore don't treat it like a schema. + if k == KEY_PROPERTIES || k == KEY_DEPENDENCIES || k == KEY_PATTERN_PROPERTIES { + if child, ok := v.(map[string]interface{}); ok { + for _, v := range child { + p.parseReferencesRecursive(v, *localRef) + } + } + } else { + p.parseReferencesRecursive(v, *localRef) + } + } + } + return nil +} + +func (p *schemaPool) GetDocument(reference gojsonreference.JsonReference) (*schemaPoolDocument, error) { + + var ( + spd *schemaPoolDocument + ok bool + err error + ) + + if internalLogEnabled { + internalLog("Get Document ( %s )", reference.String()) + } + + // Create a deep copy, so we can remove the fragment part later on without altering the original + refToUrl, _ := gojsonreference.NewJsonReference(reference.String()) + + // First check if the given fragment is a location independent identifier + // http://json-schema.org/latest/json-schema-core.html#rfc.section.8.2.3 + + if spd, ok = p.schemaPoolDocuments[refToUrl.String()]; ok { + if internalLogEnabled { + internalLog(" From pool") + } + return spd, nil + } + + // If the given reference is not a location independent identifier, + // strip the fragment and look for a document with it's base URI + + refToUrl.GetUrl().Fragment = "" + + if cachedSpd, ok := p.schemaPoolDocuments[refToUrl.String()]; ok { + document, _, err := reference.GetPointer().Get(cachedSpd.Document) + + if err != nil { + return nil, err + } + + if internalLogEnabled { + internalLog(" From pool") + } + + spd = &schemaPoolDocument{Document: document} + p.schemaPoolDocuments[reference.String()] = spd + + return spd, nil + } + + // It is not possible to load anything remotely that is not canonical... + if !reference.IsCanonical() { + return nil, errors.New(formatErrorDescription( + Locale.ReferenceMustBeCanonical(), + ErrorDetails{"reference": reference.String()}, + )) + } + + jsonReferenceLoader := p.jsonLoaderFactory.New(reference.String()) + document, err := jsonReferenceLoader.LoadJSON() + + if err != nil { + return nil, err + } + + // add the whole document to the pool for potential re-use + p.ParseReferences(document, refToUrl) + + // resolve the potential fragment and also cache it + document, _, err = reference.GetPointer().Get(document) + + if err != nil { + return nil, err + } + + return &schemaPoolDocument{Document: document}, nil +} diff --git a/tools/voluspa/vendor/github.com/xeipuuv/gojsonschema/schemaReferencePool.go b/tools/voluspa/vendor/github.com/xeipuuv/gojsonschema/schemaReferencePool.go new file mode 100644 index 00000000000..6e5e1b5cdb3 --- /dev/null +++ b/tools/voluspa/vendor/github.com/xeipuuv/gojsonschema/schemaReferencePool.go @@ -0,0 +1,68 @@ +// Copyright 2015 xeipuuv ( https://github.com/xeipuuv ) +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// author xeipuuv +// author-github https://github.com/xeipuuv +// author-mail xeipuuv@gmail.com +// +// repository-name gojsonschema +// repository-desc An implementation of JSON Schema, based on IETF's draft v4 - Go language. +// +// description Pool of referenced schemas. +// +// created 25-06-2013 + +package gojsonschema + +import ( + "fmt" +) + +type schemaReferencePool struct { + documents map[string]*subSchema +} + +func newSchemaReferencePool() *schemaReferencePool { + + p := &schemaReferencePool{} + p.documents = make(map[string]*subSchema) + + return p +} + +func (p *schemaReferencePool) Get(ref string) (r *subSchema, o bool) { + + if internalLogEnabled { + internalLog(fmt.Sprintf("Schema Reference ( %s )", ref)) + } + + if sch, ok := p.documents[ref]; ok { + if internalLogEnabled { + internalLog(fmt.Sprintf(" From pool")) + } + return sch, true + } + + return nil, false +} + +func (p *schemaReferencePool) Add(ref string, sch *subSchema) { + + if internalLogEnabled { + internalLog(fmt.Sprintf("Add Schema Reference %s to pool", ref)) + } + if _, ok := p.documents[ref]; !ok { + p.documents[ref] = sch + } +} diff --git a/tools/voluspa/vendor/github.com/xeipuuv/gojsonschema/schemaType.go b/tools/voluspa/vendor/github.com/xeipuuv/gojsonschema/schemaType.go new file mode 100644 index 00000000000..36b447a2915 --- /dev/null +++ b/tools/voluspa/vendor/github.com/xeipuuv/gojsonschema/schemaType.go @@ -0,0 +1,83 @@ +// Copyright 2015 xeipuuv ( https://github.com/xeipuuv ) +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// author xeipuuv +// author-github https://github.com/xeipuuv +// author-mail xeipuuv@gmail.com +// +// repository-name gojsonschema +// repository-desc An implementation of JSON Schema, based on IETF's draft v4 - Go language. +// +// description Helper structure to handle schema types, and the combination of them. +// +// created 28-02-2013 + +package gojsonschema + +import ( + "errors" + "fmt" + "strings" +) + +type jsonSchemaType struct { + types []string +} + +// Is the schema typed ? that is containing at least one type +// When not typed, the schema does not need any type validation +func (t *jsonSchemaType) IsTyped() bool { + return len(t.types) > 0 +} + +func (t *jsonSchemaType) Add(etype string) error { + + if !isStringInSlice(JSON_TYPES, etype) { + return errors.New(formatErrorDescription(Locale.NotAValidType(), ErrorDetails{"given": "/" + etype + "/", "expected": JSON_TYPES})) + } + + if t.Contains(etype) { + return errors.New(formatErrorDescription(Locale.Duplicated(), ErrorDetails{"type": etype})) + } + + t.types = append(t.types, etype) + + return nil +} + +func (t *jsonSchemaType) Contains(etype string) bool { + + for _, v := range t.types { + if v == etype { + return true + } + } + + return false +} + +func (t *jsonSchemaType) String() string { + + if len(t.types) == 0 { + return STRING_UNDEFINED // should never happen + } + + // Displayed as a list [type1,type2,...] + if len(t.types) > 1 { + return fmt.Sprintf("[%s]", strings.Join(t.types, ",")) + } + + // Only one type: name only + return t.types[0] +} diff --git a/tools/voluspa/vendor/github.com/xeipuuv/gojsonschema/subSchema.go b/tools/voluspa/vendor/github.com/xeipuuv/gojsonschema/subSchema.go new file mode 100644 index 00000000000..ea792a111a2 --- /dev/null +++ b/tools/voluspa/vendor/github.com/xeipuuv/gojsonschema/subSchema.go @@ -0,0 +1,255 @@ +// Copyright 2015 xeipuuv ( https://github.com/xeipuuv ) +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// author xeipuuv +// author-github https://github.com/xeipuuv +// author-mail xeipuuv@gmail.com +// +// repository-name gojsonschema +// repository-desc An implementation of JSON Schema, based on IETF's draft v4 - Go language. +// +// description Defines the structure of a sub-subSchema. +// A sub-subSchema can contain other sub-schemas. +// +// created 27-02-2013 + +package gojsonschema + +import ( + "errors" + "math/big" + "regexp" + "strings" + + "github.com/xeipuuv/gojsonreference" +) + +const ( + KEY_SCHEMA = "$schema" + KEY_ID = "id" + KEY_ID_NEW = "$id" + KEY_REF = "$ref" + KEY_TITLE = "title" + KEY_DESCRIPTION = "description" + KEY_TYPE = "type" + KEY_ITEMS = "items" + KEY_ADDITIONAL_ITEMS = "additionalItems" + KEY_PROPERTIES = "properties" + KEY_PATTERN_PROPERTIES = "patternProperties" + KEY_ADDITIONAL_PROPERTIES = "additionalProperties" + KEY_PROPERTY_NAMES = "propertyNames" + KEY_DEFINITIONS = "definitions" + KEY_MULTIPLE_OF = "multipleOf" + KEY_MINIMUM = "minimum" + KEY_MAXIMUM = "maximum" + KEY_EXCLUSIVE_MINIMUM = "exclusiveMinimum" + KEY_EXCLUSIVE_MAXIMUM = "exclusiveMaximum" + KEY_MIN_LENGTH = "minLength" + KEY_MAX_LENGTH = "maxLength" + KEY_PATTERN = "pattern" + KEY_FORMAT = "format" + KEY_MIN_PROPERTIES = "minProperties" + KEY_MAX_PROPERTIES = "maxProperties" + KEY_DEPENDENCIES = "dependencies" + KEY_REQUIRED = "required" + KEY_MIN_ITEMS = "minItems" + KEY_MAX_ITEMS = "maxItems" + KEY_UNIQUE_ITEMS = "uniqueItems" + KEY_CONTAINS = "contains" + KEY_CONST = "const" + KEY_ENUM = "enum" + KEY_ONE_OF = "oneOf" + KEY_ANY_OF = "anyOf" + KEY_ALL_OF = "allOf" + KEY_NOT = "not" + KEY_IF = "if" + KEY_THEN = "then" + KEY_ELSE = "else" +) + +type subSchema struct { + + // basic subSchema meta properties + id *gojsonreference.JsonReference + title *string + description *string + + property string + + // Types associated with the subSchema + types jsonSchemaType + + // Reference url + ref *gojsonreference.JsonReference + // Schema referenced + refSchema *subSchema + + // hierarchy + parent *subSchema + itemsChildren []*subSchema + itemsChildrenIsSingleSchema bool + propertiesChildren []*subSchema + + // validation : number / integer + multipleOf *big.Float + maximum *big.Float + exclusiveMaximum bool + minimum *big.Float + exclusiveMinimum bool + + // validation : string + minLength *int + maxLength *int + pattern *regexp.Regexp + format string + + // validation : object + minProperties *int + maxProperties *int + required []string + + dependencies map[string]interface{} + additionalProperties interface{} + patternProperties map[string]*subSchema + propertyNames *subSchema + + // validation : array + minItems *int + maxItems *int + uniqueItems bool + contains *subSchema + + additionalItems interface{} + + // validation : all + _const *string //const is a golang keyword + enum []string + + // validation : subSchema + oneOf []*subSchema + anyOf []*subSchema + allOf []*subSchema + not *subSchema + _if *subSchema // if/else are golang keywords + _then *subSchema + _else *subSchema +} + +func (s *subSchema) AddConst(i interface{}) error { + + is, err := marshalWithoutNumber(i) + if err != nil { + return err + } + s._const = is + return nil +} + +func (s *subSchema) AddEnum(i interface{}) error { + + is, err := marshalWithoutNumber(i) + if err != nil { + return err + } + + if isStringInSlice(s.enum, *is) { + return errors.New(formatErrorDescription( + Locale.KeyItemsMustBeUnique(), + ErrorDetails{"key": KEY_ENUM}, + )) + } + + s.enum = append(s.enum, *is) + + return nil +} + +func (s *subSchema) ContainsEnum(i interface{}) (bool, error) { + + is, err := marshalWithoutNumber(i) + if err != nil { + return false, err + } + + return isStringInSlice(s.enum, *is), nil +} + +func (s *subSchema) AddOneOf(subSchema *subSchema) { + s.oneOf = append(s.oneOf, subSchema) +} + +func (s *subSchema) AddAllOf(subSchema *subSchema) { + s.allOf = append(s.allOf, subSchema) +} + +func (s *subSchema) AddAnyOf(subSchema *subSchema) { + s.anyOf = append(s.anyOf, subSchema) +} + +func (s *subSchema) SetNot(subSchema *subSchema) { + s.not = subSchema +} + +func (s *subSchema) SetIf(subSchema *subSchema) { + s._if = subSchema +} + +func (s *subSchema) SetThen(subSchema *subSchema) { + s._then = subSchema +} + +func (s *subSchema) SetElse(subSchema *subSchema) { + s._else = subSchema +} + +func (s *subSchema) AddRequired(value string) error { + + if isStringInSlice(s.required, value) { + return errors.New(formatErrorDescription( + Locale.KeyItemsMustBeUnique(), + ErrorDetails{"key": KEY_REQUIRED}, + )) + } + + s.required = append(s.required, value) + + return nil +} + +func (s *subSchema) AddItemsChild(child *subSchema) { + s.itemsChildren = append(s.itemsChildren, child) +} + +func (s *subSchema) AddPropertiesChild(child *subSchema) { + s.propertiesChildren = append(s.propertiesChildren, child) +} + +func (s *subSchema) PatternPropertiesString() string { + + if s.patternProperties == nil || len(s.patternProperties) == 0 { + return STRING_UNDEFINED // should never happen + } + + patternPropertiesKeySlice := []string{} + for pk := range s.patternProperties { + patternPropertiesKeySlice = append(patternPropertiesKeySlice, `"`+pk+`"`) + } + + if len(patternPropertiesKeySlice) == 1 { + return patternPropertiesKeySlice[0] + } + + return "[" + strings.Join(patternPropertiesKeySlice, ",") + "]" + +} diff --git a/tools/voluspa/vendor/github.com/xeipuuv/gojsonschema/types.go b/tools/voluspa/vendor/github.com/xeipuuv/gojsonschema/types.go new file mode 100644 index 00000000000..952d22ef65e --- /dev/null +++ b/tools/voluspa/vendor/github.com/xeipuuv/gojsonschema/types.go @@ -0,0 +1,58 @@ +// Copyright 2015 xeipuuv ( https://github.com/xeipuuv ) +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// author xeipuuv +// author-github https://github.com/xeipuuv +// author-mail xeipuuv@gmail.com +// +// repository-name gojsonschema +// repository-desc An implementation of JSON Schema, based on IETF's draft v4 - Go language. +// +// description Contains const types for schema and JSON. +// +// created 28-02-2013 + +package gojsonschema + +const ( + TYPE_ARRAY = `array` + TYPE_BOOLEAN = `boolean` + TYPE_INTEGER = `integer` + TYPE_NUMBER = `number` + TYPE_NULL = `null` + TYPE_OBJECT = `object` + TYPE_STRING = `string` +) + +var JSON_TYPES []string +var SCHEMA_TYPES []string + +func init() { + JSON_TYPES = []string{ + TYPE_ARRAY, + TYPE_BOOLEAN, + TYPE_INTEGER, + TYPE_NUMBER, + TYPE_NULL, + TYPE_OBJECT, + TYPE_STRING} + + SCHEMA_TYPES = []string{ + TYPE_ARRAY, + TYPE_BOOLEAN, + TYPE_INTEGER, + TYPE_NUMBER, + TYPE_OBJECT, + TYPE_STRING} +} diff --git a/tools/voluspa/vendor/github.com/xeipuuv/gojsonschema/utils.go b/tools/voluspa/vendor/github.com/xeipuuv/gojsonschema/utils.go new file mode 100644 index 00000000000..cc3100a78dd --- /dev/null +++ b/tools/voluspa/vendor/github.com/xeipuuv/gojsonschema/utils.go @@ -0,0 +1,226 @@ +// Copyright 2015 xeipuuv ( https://github.com/xeipuuv ) +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// author xeipuuv +// author-github https://github.com/xeipuuv +// author-mail xeipuuv@gmail.com +// +// repository-name gojsonschema +// repository-desc An implementation of JSON Schema, based on IETF's draft v4 - Go language. +// +// description Various utility functions. +// +// created 26-02-2013 + +package gojsonschema + +import ( + "encoding/json" + "fmt" + "math" + "math/big" + "reflect" +) + +func isKind(what interface{}, kinds ...reflect.Kind) bool { + target := what + if isJsonNumber(what) { + // JSON Numbers are strings! + target = *mustBeNumber(what) + } + targetKind := reflect.ValueOf(target).Kind() + for _, kind := range kinds { + if targetKind == kind { + return true + } + } + return false +} + +func existsMapKey(m map[string]interface{}, k string) bool { + _, ok := m[k] + return ok +} + +func isStringInSlice(s []string, what string) bool { + for i := range s { + if s[i] == what { + return true + } + } + return false +} + +func marshalToJsonString(value interface{}) (*string, error) { + + mBytes, err := json.Marshal(value) + if err != nil { + return nil, err + } + + sBytes := string(mBytes) + return &sBytes, nil +} + +func marshalWithoutNumber(value interface{}) (*string, error) { + + // The JSON is decoded using https://golang.org/pkg/encoding/json/#Decoder.UseNumber + // This means the numbers are internally still represented as strings and therefore 1.00 is unequal to 1 + // One way to eliminate these differences is to decode and encode the JSON one more time without Decoder.UseNumber + // so that these differences in representation are removed + + jsonString, err := marshalToJsonString(value) + if err != nil { + return nil, err + } + + var document interface{} + + err = json.Unmarshal([]byte(*jsonString), &document) + if err != nil { + return nil, err + } + + return marshalToJsonString(document) +} + +func isJsonNumber(what interface{}) bool { + + switch what.(type) { + + case json.Number: + return true + } + + return false +} + +func checkJsonInteger(what interface{}) (isInt bool) { + + jsonNumber := what.(json.Number) + + bigFloat, isValidNumber := new(big.Float).SetString(string(jsonNumber)) + + return isValidNumber && bigFloat.IsInt() + +} + +// same as ECMA Number.MAX_SAFE_INTEGER and Number.MIN_SAFE_INTEGER +const ( + max_json_float = float64(1<<53 - 1) // 9007199254740991.0 2^53 - 1 + min_json_float = -float64(1<<53 - 1) //-9007199254740991.0 -2^53 - 1 +) + +func isFloat64AnInteger(f float64) bool { + + if math.IsNaN(f) || math.IsInf(f, 0) || f < min_json_float || f > max_json_float { + return false + } + + return f == float64(int64(f)) || f == float64(uint64(f)) +} + +func mustBeInteger(what interface{}) *int { + + if isJsonNumber(what) { + + number := what.(json.Number) + + isInt := checkJsonInteger(number) + + if isInt { + + int64Value, err := number.Int64() + if err != nil { + return nil + } + + int32Value := int(int64Value) + return &int32Value + + } else { + return nil + } + + } + + return nil +} + +func mustBeNumber(what interface{}) *big.Float { + + if isJsonNumber(what) { + number := what.(json.Number) + float64Value, success := new(big.Float).SetString(string(number)) + if success { + return float64Value + } else { + return nil + } + + } + + return nil + +} + +// formats a number so that it is displayed as the smallest string possible +func resultErrorFormatJsonNumber(n json.Number) string { + + if int64Value, err := n.Int64(); err == nil { + return fmt.Sprintf("%d", int64Value) + } + + float64Value, _ := n.Float64() + + return fmt.Sprintf("%g", float64Value) +} + +// formats a number so that it is displayed as the smallest string possible +func resultErrorFormatNumber(n float64) string { + + if isFloat64AnInteger(n) { + return fmt.Sprintf("%d", int64(n)) + } + + return fmt.Sprintf("%g", n) +} + +func convertDocumentNode(val interface{}) interface{} { + + if lval, ok := val.([]interface{}); ok { + + res := []interface{}{} + for _, v := range lval { + res = append(res, convertDocumentNode(v)) + } + + return res + + } + + if mval, ok := val.(map[interface{}]interface{}); ok { + + res := map[string]interface{}{} + + for k, v := range mval { + res[k.(string)] = convertDocumentNode(v) + } + + return res + + } + + return val +} diff --git a/tools/voluspa/vendor/github.com/xeipuuv/gojsonschema/validation.go b/tools/voluspa/vendor/github.com/xeipuuv/gojsonschema/validation.go new file mode 100644 index 00000000000..2f1e7df36c1 --- /dev/null +++ b/tools/voluspa/vendor/github.com/xeipuuv/gojsonschema/validation.go @@ -0,0 +1,928 @@ +// Copyright 2015 xeipuuv ( https://github.com/xeipuuv ) +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// author xeipuuv +// author-github https://github.com/xeipuuv +// author-mail xeipuuv@gmail.com +// +// repository-name gojsonschema +// repository-desc An implementation of JSON Schema, based on IETF's draft v4 - Go language. +// +// description Extends Schema and subSchema, implements the validation phase. +// +// created 28-02-2013 + +package gojsonschema + +import ( + "encoding/json" + "math/big" + "reflect" + "regexp" + "strconv" + "strings" + "unicode/utf8" +) + +func Validate(ls JSONLoader, ld JSONLoader) (*Result, error) { + + var err error + + // load schema + + schema, err := NewSchema(ls) + if err != nil { + return nil, err + } + + // begine validation + + return schema.Validate(ld) + +} + +func (v *Schema) Validate(l JSONLoader) (*Result, error) { + + // load document + + root, err := l.LoadJSON() + if err != nil { + return nil, err + } + + return v.validateDocument(root), nil +} + +func (v *Schema) validateDocument(root interface{}) *Result { + // begin validation + + result := &Result{} + context := NewJsonContext(STRING_CONTEXT_ROOT, nil) + v.rootSchema.validateRecursive(v.rootSchema, root, result, context) + + return result +} + +func (v *subSchema) subValidateWithContext(document interface{}, context *JsonContext) *Result { + result := &Result{} + v.validateRecursive(v, document, result, context) + return result +} + +// Walker function to validate the json recursively against the subSchema +func (v *subSchema) validateRecursive(currentSubSchema *subSchema, currentNode interface{}, result *Result, context *JsonContext) { + + if internalLogEnabled { + internalLog("validateRecursive %s", context.String()) + internalLog(" %v", currentNode) + } + + // Handle referenced schemas, returns directly when a $ref is found + if currentSubSchema.refSchema != nil { + v.validateRecursive(currentSubSchema.refSchema, currentNode, result, context) + return + } + + // Check for null value + if currentNode == nil { + if currentSubSchema.types.IsTyped() && !currentSubSchema.types.Contains(TYPE_NULL) { + result.addInternalError( + new(InvalidTypeError), + context, + currentNode, + ErrorDetails{ + "expected": currentSubSchema.types.String(), + "given": TYPE_NULL, + }, + ) + return + } + + currentSubSchema.validateSchema(currentSubSchema, currentNode, result, context) + v.validateCommon(currentSubSchema, currentNode, result, context) + + } else { // Not a null value + + if isJsonNumber(currentNode) { + + value := currentNode.(json.Number) + + isInt := checkJsonInteger(value) + + validType := currentSubSchema.types.Contains(TYPE_NUMBER) || (isInt && currentSubSchema.types.Contains(TYPE_INTEGER)) + + if currentSubSchema.types.IsTyped() && !validType { + + givenType := TYPE_INTEGER + if !isInt { + givenType = TYPE_NUMBER + } + + result.addInternalError( + new(InvalidTypeError), + context, + currentNode, + ErrorDetails{ + "expected": currentSubSchema.types.String(), + "given": givenType, + }, + ) + return + } + + currentSubSchema.validateSchema(currentSubSchema, value, result, context) + v.validateNumber(currentSubSchema, value, result, context) + v.validateCommon(currentSubSchema, value, result, context) + v.validateString(currentSubSchema, value, result, context) + + } else { + + rValue := reflect.ValueOf(currentNode) + rKind := rValue.Kind() + + switch rKind { + + // Slice => JSON array + + case reflect.Slice: + + if currentSubSchema.types.IsTyped() && !currentSubSchema.types.Contains(TYPE_ARRAY) { + result.addInternalError( + new(InvalidTypeError), + context, + currentNode, + ErrorDetails{ + "expected": currentSubSchema.types.String(), + "given": TYPE_ARRAY, + }, + ) + return + } + + castCurrentNode := currentNode.([]interface{}) + + currentSubSchema.validateSchema(currentSubSchema, castCurrentNode, result, context) + + v.validateArray(currentSubSchema, castCurrentNode, result, context) + v.validateCommon(currentSubSchema, castCurrentNode, result, context) + + // Map => JSON object + + case reflect.Map: + if currentSubSchema.types.IsTyped() && !currentSubSchema.types.Contains(TYPE_OBJECT) { + result.addInternalError( + new(InvalidTypeError), + context, + currentNode, + ErrorDetails{ + "expected": currentSubSchema.types.String(), + "given": TYPE_OBJECT, + }, + ) + return + } + + castCurrentNode, ok := currentNode.(map[string]interface{}) + if !ok { + castCurrentNode = convertDocumentNode(currentNode).(map[string]interface{}) + } + + currentSubSchema.validateSchema(currentSubSchema, castCurrentNode, result, context) + + v.validateObject(currentSubSchema, castCurrentNode, result, context) + v.validateCommon(currentSubSchema, castCurrentNode, result, context) + + for _, pSchema := range currentSubSchema.propertiesChildren { + nextNode, ok := castCurrentNode[pSchema.property] + if ok { + subContext := NewJsonContext(pSchema.property, context) + v.validateRecursive(pSchema, nextNode, result, subContext) + } + } + + // Simple JSON values : string, number, boolean + + case reflect.Bool: + + if currentSubSchema.types.IsTyped() && !currentSubSchema.types.Contains(TYPE_BOOLEAN) { + result.addInternalError( + new(InvalidTypeError), + context, + currentNode, + ErrorDetails{ + "expected": currentSubSchema.types.String(), + "given": TYPE_BOOLEAN, + }, + ) + return + } + + value := currentNode.(bool) + + currentSubSchema.validateSchema(currentSubSchema, value, result, context) + v.validateNumber(currentSubSchema, value, result, context) + v.validateCommon(currentSubSchema, value, result, context) + v.validateString(currentSubSchema, value, result, context) + + case reflect.String: + + if currentSubSchema.types.IsTyped() && !currentSubSchema.types.Contains(TYPE_STRING) { + result.addInternalError( + new(InvalidTypeError), + context, + currentNode, + ErrorDetails{ + "expected": currentSubSchema.types.String(), + "given": TYPE_STRING, + }, + ) + return + } + + value := currentNode.(string) + + currentSubSchema.validateSchema(currentSubSchema, value, result, context) + v.validateNumber(currentSubSchema, value, result, context) + v.validateCommon(currentSubSchema, value, result, context) + v.validateString(currentSubSchema, value, result, context) + + } + + } + + } + + result.incrementScore() +} + +// Different kinds of validation there, subSchema / common / array / object / string... +func (v *subSchema) validateSchema(currentSubSchema *subSchema, currentNode interface{}, result *Result, context *JsonContext) { + + if internalLogEnabled { + internalLog("validateSchema %s", context.String()) + internalLog(" %v", currentNode) + } + + if len(currentSubSchema.anyOf) > 0 { + + validatedAnyOf := false + var bestValidationResult *Result + + for _, anyOfSchema := range currentSubSchema.anyOf { + if !validatedAnyOf { + validationResult := anyOfSchema.subValidateWithContext(currentNode, context) + validatedAnyOf = validationResult.Valid() + + if !validatedAnyOf && (bestValidationResult == nil || validationResult.score > bestValidationResult.score) { + bestValidationResult = validationResult + } + } + } + if !validatedAnyOf { + + result.addInternalError(new(NumberAnyOfError), context, currentNode, ErrorDetails{}) + + if bestValidationResult != nil { + // add error messages of closest matching subSchema as + // that's probably the one the user was trying to match + result.mergeErrors(bestValidationResult) + } + } + } + + if len(currentSubSchema.oneOf) > 0 { + + nbValidated := 0 + var bestValidationResult *Result + + for _, oneOfSchema := range currentSubSchema.oneOf { + validationResult := oneOfSchema.subValidateWithContext(currentNode, context) + if validationResult.Valid() { + nbValidated++ + } else if nbValidated == 0 && (bestValidationResult == nil || validationResult.score > bestValidationResult.score) { + bestValidationResult = validationResult + } + } + + if nbValidated != 1 { + + result.addInternalError(new(NumberOneOfError), context, currentNode, ErrorDetails{}) + + if nbValidated == 0 { + // add error messages of closest matching subSchema as + // that's probably the one the user was trying to match + result.mergeErrors(bestValidationResult) + } + } + + } + + if len(currentSubSchema.allOf) > 0 { + nbValidated := 0 + + for _, allOfSchema := range currentSubSchema.allOf { + validationResult := allOfSchema.subValidateWithContext(currentNode, context) + if validationResult.Valid() { + nbValidated++ + } + result.mergeErrors(validationResult) + } + + if nbValidated != len(currentSubSchema.allOf) { + result.addInternalError(new(NumberAllOfError), context, currentNode, ErrorDetails{}) + } + } + + if currentSubSchema.not != nil { + validationResult := currentSubSchema.not.subValidateWithContext(currentNode, context) + if validationResult.Valid() { + result.addInternalError(new(NumberNotError), context, currentNode, ErrorDetails{}) + } + } + + if currentSubSchema.dependencies != nil && len(currentSubSchema.dependencies) > 0 { + if isKind(currentNode, reflect.Map) { + for elementKey := range currentNode.(map[string]interface{}) { + if dependency, ok := currentSubSchema.dependencies[elementKey]; ok { + switch dependency := dependency.(type) { + + case []string: + for _, dependOnKey := range dependency { + if _, dependencyResolved := currentNode.(map[string]interface{})[dependOnKey]; !dependencyResolved { + result.addInternalError( + new(MissingDependencyError), + context, + currentNode, + ErrorDetails{"dependency": dependOnKey}, + ) + } + } + + case *subSchema: + dependency.validateRecursive(dependency, currentNode, result, context) + } + } + } + } + } + + if currentSubSchema._if != nil { + validationResultIf := currentSubSchema._if.subValidateWithContext(currentNode, context) + if currentSubSchema._then != nil && validationResultIf.Valid() { + validationResultThen := currentSubSchema._then.subValidateWithContext(currentNode, context) + if !validationResultThen.Valid() { + result.addInternalError(new(ConditionThenError), context, currentNode, ErrorDetails{}) + result.mergeErrors(validationResultThen) + } + } + if currentSubSchema._else != nil && !validationResultIf.Valid() { + validationResultElse := currentSubSchema._else.subValidateWithContext(currentNode, context) + if !validationResultElse.Valid() { + result.addInternalError(new(ConditionElseError), context, currentNode, ErrorDetails{}) + result.mergeErrors(validationResultElse) + } + } + } + + result.incrementScore() +} + +func (v *subSchema) validateCommon(currentSubSchema *subSchema, value interface{}, result *Result, context *JsonContext) { + + if internalLogEnabled { + internalLog("validateCommon %s", context.String()) + internalLog(" %v", value) + } + + // const: + if currentSubSchema._const != nil { + vString, err := marshalWithoutNumber(value) + if err != nil { + result.addInternalError(new(InternalError), context, value, ErrorDetails{"error": err}) + } + if *vString != *currentSubSchema._const { + result.addInternalError(new(ConstError), + context, + value, + ErrorDetails{ + "allowed": *currentSubSchema._const, + }, + ) + } + } + + // enum: + if len(currentSubSchema.enum) > 0 { + has, err := currentSubSchema.ContainsEnum(value) + if err != nil { + result.addInternalError(new(InternalError), context, value, ErrorDetails{"error": err}) + } + if !has { + result.addInternalError( + new(EnumError), + context, + value, + ErrorDetails{ + "allowed": strings.Join(currentSubSchema.enum, ", "), + }, + ) + } + } + + result.incrementScore() +} + +func (v *subSchema) validateArray(currentSubSchema *subSchema, value []interface{}, result *Result, context *JsonContext) { + + if internalLogEnabled { + internalLog("validateArray %s", context.String()) + internalLog(" %v", value) + } + + nbValues := len(value) + + // TODO explain + if currentSubSchema.itemsChildrenIsSingleSchema { + for i := range value { + subContext := NewJsonContext(strconv.Itoa(i), context) + validationResult := currentSubSchema.itemsChildren[0].subValidateWithContext(value[i], subContext) + result.mergeErrors(validationResult) + } + } else { + if currentSubSchema.itemsChildren != nil && len(currentSubSchema.itemsChildren) > 0 { + + nbItems := len(currentSubSchema.itemsChildren) + + // while we have both schemas and values, check them against each other + for i := 0; i != nbItems && i != nbValues; i++ { + subContext := NewJsonContext(strconv.Itoa(i), context) + validationResult := currentSubSchema.itemsChildren[i].subValidateWithContext(value[i], subContext) + result.mergeErrors(validationResult) + } + + if nbItems < nbValues { + // we have less schemas than elements in the instance array, + // but that might be ok if "additionalItems" is specified. + + switch currentSubSchema.additionalItems.(type) { + case bool: + if !currentSubSchema.additionalItems.(bool) { + result.addInternalError(new(ArrayNoAdditionalItemsError), context, value, ErrorDetails{}) + } + case *subSchema: + additionalItemSchema := currentSubSchema.additionalItems.(*subSchema) + for i := nbItems; i != nbValues; i++ { + subContext := NewJsonContext(strconv.Itoa(i), context) + validationResult := additionalItemSchema.subValidateWithContext(value[i], subContext) + result.mergeErrors(validationResult) + } + } + } + } + } + + // minItems & maxItems + if currentSubSchema.minItems != nil { + if nbValues < int(*currentSubSchema.minItems) { + result.addInternalError( + new(ArrayMinItemsError), + context, + value, + ErrorDetails{"min": *currentSubSchema.minItems}, + ) + } + } + if currentSubSchema.maxItems != nil { + if nbValues > int(*currentSubSchema.maxItems) { + result.addInternalError( + new(ArrayMaxItemsError), + context, + value, + ErrorDetails{"max": *currentSubSchema.maxItems}, + ) + } + } + + // uniqueItems: + if currentSubSchema.uniqueItems { + var stringifiedItems []string + for _, v := range value { + vString, err := marshalWithoutNumber(v) + if err != nil { + result.addInternalError(new(InternalError), context, value, ErrorDetails{"err": err}) + } + if isStringInSlice(stringifiedItems, *vString) { + result.addInternalError( + new(ItemsMustBeUniqueError), + context, + value, + ErrorDetails{"type": TYPE_ARRAY}, + ) + } + stringifiedItems = append(stringifiedItems, *vString) + } + } + + // contains: + + if currentSubSchema.contains != nil { + validatedOne := false + var bestValidationResult *Result + + for i, v := range value { + subContext := NewJsonContext(strconv.Itoa(i), context) + + validationResult := currentSubSchema.contains.subValidateWithContext(v, subContext) + if validationResult.Valid() { + validatedOne = true + break + } else { + if bestValidationResult == nil || validationResult.score > bestValidationResult.score { + bestValidationResult = validationResult + } + } + } + if !validatedOne { + result.addInternalError( + new(ArrayContainsError), + context, + value, + ErrorDetails{}, + ) + if bestValidationResult != nil { + result.mergeErrors(bestValidationResult) + } + } + } + + result.incrementScore() +} + +func (v *subSchema) validateObject(currentSubSchema *subSchema, value map[string]interface{}, result *Result, context *JsonContext) { + + if internalLogEnabled { + internalLog("validateObject %s", context.String()) + internalLog(" %v", value) + } + + // minProperties & maxProperties: + if currentSubSchema.minProperties != nil { + if len(value) < int(*currentSubSchema.minProperties) { + result.addInternalError( + new(ArrayMinPropertiesError), + context, + value, + ErrorDetails{"min": *currentSubSchema.minProperties}, + ) + } + } + if currentSubSchema.maxProperties != nil { + if len(value) > int(*currentSubSchema.maxProperties) { + result.addInternalError( + new(ArrayMaxPropertiesError), + context, + value, + ErrorDetails{"max": *currentSubSchema.maxProperties}, + ) + } + } + + // required: + for _, requiredProperty := range currentSubSchema.required { + _, ok := value[requiredProperty] + if ok { + result.incrementScore() + } else { + result.addInternalError( + new(RequiredError), + context, + value, + ErrorDetails{"property": requiredProperty}, + ) + } + } + + // additionalProperty & patternProperty: + if currentSubSchema.additionalProperties != nil { + + switch currentSubSchema.additionalProperties.(type) { + case bool: + + if !currentSubSchema.additionalProperties.(bool) { + + for pk := range value { + + found := false + for _, spValue := range currentSubSchema.propertiesChildren { + if pk == spValue.property { + found = true + } + } + + pp_has, pp_match := v.validatePatternProperty(currentSubSchema, pk, value[pk], result, context) + + if found { + + if pp_has && !pp_match { + result.addInternalError( + new(AdditionalPropertyNotAllowedError), + context, + value[pk], + ErrorDetails{"property": pk}, + ) + } + + } else { + + if !pp_has || !pp_match { + result.addInternalError( + new(AdditionalPropertyNotAllowedError), + context, + value[pk], + ErrorDetails{"property": pk}, + ) + } + + } + } + } + + case *subSchema: + + additionalPropertiesSchema := currentSubSchema.additionalProperties.(*subSchema) + for pk := range value { + + found := false + for _, spValue := range currentSubSchema.propertiesChildren { + if pk == spValue.property { + found = true + } + } + + pp_has, pp_match := v.validatePatternProperty(currentSubSchema, pk, value[pk], result, context) + + if found { + + if pp_has && !pp_match { + validationResult := additionalPropertiesSchema.subValidateWithContext(value[pk], context) + result.mergeErrors(validationResult) + } + + } else { + + if !pp_has || !pp_match { + validationResult := additionalPropertiesSchema.subValidateWithContext(value[pk], context) + result.mergeErrors(validationResult) + } + + } + + } + } + } else { + + for pk := range value { + + pp_has, pp_match := v.validatePatternProperty(currentSubSchema, pk, value[pk], result, context) + + if pp_has && !pp_match { + + result.addInternalError( + new(InvalidPropertyPatternError), + context, + value[pk], + ErrorDetails{ + "property": pk, + "pattern": currentSubSchema.PatternPropertiesString(), + }, + ) + } + + } + } + + // propertyNames: + if currentSubSchema.propertyNames != nil { + for pk := range value { + validationResult := currentSubSchema.propertyNames.subValidateWithContext(pk, context) + if !validationResult.Valid() { + result.addInternalError(new(InvalidPropertyNameError), + context, + value, ErrorDetails{ + "property": pk, + }) + result.mergeErrors(validationResult) + } + } + } + + result.incrementScore() +} + +func (v *subSchema) validatePatternProperty(currentSubSchema *subSchema, key string, value interface{}, result *Result, context *JsonContext) (has bool, matched bool) { + + if internalLogEnabled { + internalLog("validatePatternProperty %s", context.String()) + internalLog(" %s %v", key, value) + } + + has = false + + validatedkey := false + + for pk, pv := range currentSubSchema.patternProperties { + if matches, _ := regexp.MatchString(pk, key); matches { + has = true + subContext := NewJsonContext(key, context) + validationResult := pv.subValidateWithContext(value, subContext) + result.mergeErrors(validationResult) + validatedkey = true + } + } + + if !validatedkey { + return has, false + } + + result.incrementScore() + + return has, true +} + +func (v *subSchema) validateString(currentSubSchema *subSchema, value interface{}, result *Result, context *JsonContext) { + + // Ignore JSON numbers + if isJsonNumber(value) { + return + } + + // Ignore non strings + if !isKind(value, reflect.String) { + return + } + + if internalLogEnabled { + internalLog("validateString %s", context.String()) + internalLog(" %v", value) + } + + stringValue := value.(string) + + // minLength & maxLength: + if currentSubSchema.minLength != nil { + if utf8.RuneCount([]byte(stringValue)) < int(*currentSubSchema.minLength) { + result.addInternalError( + new(StringLengthGTEError), + context, + value, + ErrorDetails{"min": *currentSubSchema.minLength}, + ) + } + } + if currentSubSchema.maxLength != nil { + if utf8.RuneCount([]byte(stringValue)) > int(*currentSubSchema.maxLength) { + result.addInternalError( + new(StringLengthLTEError), + context, + value, + ErrorDetails{"max": *currentSubSchema.maxLength}, + ) + } + } + + // pattern: + if currentSubSchema.pattern != nil { + if !currentSubSchema.pattern.MatchString(stringValue) { + result.addInternalError( + new(DoesNotMatchPatternError), + context, + value, + ErrorDetails{"pattern": currentSubSchema.pattern}, + ) + + } + } + + // format + if currentSubSchema.format != "" { + if !FormatCheckers.IsFormat(currentSubSchema.format, stringValue) { + result.addInternalError( + new(DoesNotMatchFormatError), + context, + value, + ErrorDetails{"format": currentSubSchema.format}, + ) + } + } + + result.incrementScore() +} + +func (v *subSchema) validateNumber(currentSubSchema *subSchema, value interface{}, result *Result, context *JsonContext) { + + // Ignore non numbers + if !isJsonNumber(value) { + return + } + + if internalLogEnabled { + internalLog("validateNumber %s", context.String()) + internalLog(" %v", value) + } + + number := value.(json.Number) + float64Value, _ := new(big.Float).SetString(string(number)) + + // multipleOf: + if currentSubSchema.multipleOf != nil { + + if q := new(big.Float).Quo(float64Value, currentSubSchema.multipleOf); !q.IsInt() { + result.addInternalError( + new(MultipleOfError), + context, + resultErrorFormatJsonNumber(number), + ErrorDetails{"multiple": currentSubSchema.multipleOf}, + ) + } + } + + //maximum & exclusiveMaximum: + if currentSubSchema.maximum != nil { + if currentSubSchema.exclusiveMaximum { + if float64Value.Cmp(currentSubSchema.maximum) >= 0 { + result.addInternalError( + new(NumberLTError), + context, + resultErrorFormatJsonNumber(number), + ErrorDetails{ + "max": currentSubSchema.maximum, + }, + ) + } + } else { + if float64Value.Cmp(currentSubSchema.maximum) == 1 { + result.addInternalError( + new(NumberLTEError), + context, + resultErrorFormatJsonNumber(number), + ErrorDetails{ + "max": currentSubSchema.maximum, + }, + ) + } + } + } + + //minimum & exclusiveMinimum: + if currentSubSchema.minimum != nil { + if currentSubSchema.exclusiveMinimum { + if float64Value.Cmp(currentSubSchema.minimum) <= 0 { + // if float64Value <= *currentSubSchema.minimum { + result.addInternalError( + new(NumberGTError), + context, + resultErrorFormatJsonNumber(number), + ErrorDetails{ + "min": currentSubSchema.minimum, + }, + ) + } + } else { + if float64Value.Cmp(currentSubSchema.minimum) == -1 { + result.addInternalError( + new(NumberGTEError), + context, + resultErrorFormatJsonNumber(number), + ErrorDetails{ + "min": currentSubSchema.minimum, + }, + ) + } + } + } + + // format + if currentSubSchema.format != "" { + if !FormatCheckers.IsFormat(currentSubSchema.format, float64Value) { + result.addInternalError( + new(DoesNotMatchFormatError), + context, + value, + ErrorDetails{"format": currentSubSchema.format}, + ) + } + } + + result.incrementScore() +} diff --git a/tools/voluspa/vendor/gopkg.in/yaml.v2/LICENSE b/tools/voluspa/vendor/gopkg.in/yaml.v2/LICENSE new file mode 100644 index 00000000000..8dada3edaf5 --- /dev/null +++ b/tools/voluspa/vendor/gopkg.in/yaml.v2/LICENSE @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "{}" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright {yyyy} {name of copyright owner} + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/tools/voluspa/vendor/gopkg.in/yaml.v2/LICENSE.libyaml b/tools/voluspa/vendor/gopkg.in/yaml.v2/LICENSE.libyaml new file mode 100644 index 00000000000..8da58fbf6f8 --- /dev/null +++ b/tools/voluspa/vendor/gopkg.in/yaml.v2/LICENSE.libyaml @@ -0,0 +1,31 @@ +The following files were ported to Go from C files of libyaml, and thus +are still covered by their original copyright and license: + + apic.go + emitterc.go + parserc.go + readerc.go + scannerc.go + writerc.go + yamlh.go + yamlprivateh.go + +Copyright (c) 2006 Kirill Simonov + +Permission is hereby granted, free of charge, to any person obtaining a copy of +this software and associated documentation files (the "Software"), to deal in +the Software without restriction, including without limitation the rights to +use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies +of the Software, and to permit persons to whom the Software is furnished to do +so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/tools/voluspa/vendor/gopkg.in/yaml.v2/apic.go b/tools/voluspa/vendor/gopkg.in/yaml.v2/apic.go new file mode 100644 index 00000000000..1f7e87e6727 --- /dev/null +++ b/tools/voluspa/vendor/gopkg.in/yaml.v2/apic.go @@ -0,0 +1,739 @@ +package yaml + +import ( + "io" +) + +func yaml_insert_token(parser *yaml_parser_t, pos int, token *yaml_token_t) { + //fmt.Println("yaml_insert_token", "pos:", pos, "typ:", token.typ, "head:", parser.tokens_head, "len:", len(parser.tokens)) + + // Check if we can move the queue at the beginning of the buffer. + if parser.tokens_head > 0 && len(parser.tokens) == cap(parser.tokens) { + if parser.tokens_head != len(parser.tokens) { + copy(parser.tokens, parser.tokens[parser.tokens_head:]) + } + parser.tokens = parser.tokens[:len(parser.tokens)-parser.tokens_head] + parser.tokens_head = 0 + } + parser.tokens = append(parser.tokens, *token) + if pos < 0 { + return + } + copy(parser.tokens[parser.tokens_head+pos+1:], parser.tokens[parser.tokens_head+pos:]) + parser.tokens[parser.tokens_head+pos] = *token +} + +// Create a new parser object. +func yaml_parser_initialize(parser *yaml_parser_t) bool { + *parser = yaml_parser_t{ + raw_buffer: make([]byte, 0, input_raw_buffer_size), + buffer: make([]byte, 0, input_buffer_size), + } + return true +} + +// Destroy a parser object. +func yaml_parser_delete(parser *yaml_parser_t) { + *parser = yaml_parser_t{} +} + +// String read handler. +func yaml_string_read_handler(parser *yaml_parser_t, buffer []byte) (n int, err error) { + if parser.input_pos == len(parser.input) { + return 0, io.EOF + } + n = copy(buffer, parser.input[parser.input_pos:]) + parser.input_pos += n + return n, nil +} + +// Reader read handler. +func yaml_reader_read_handler(parser *yaml_parser_t, buffer []byte) (n int, err error) { + return parser.input_reader.Read(buffer) +} + +// Set a string input. +func yaml_parser_set_input_string(parser *yaml_parser_t, input []byte) { + if parser.read_handler != nil { + panic("must set the input source only once") + } + parser.read_handler = yaml_string_read_handler + parser.input = input + parser.input_pos = 0 +} + +// Set a file input. +func yaml_parser_set_input_reader(parser *yaml_parser_t, r io.Reader) { + if parser.read_handler != nil { + panic("must set the input source only once") + } + parser.read_handler = yaml_reader_read_handler + parser.input_reader = r +} + +// Set the source encoding. +func yaml_parser_set_encoding(parser *yaml_parser_t, encoding yaml_encoding_t) { + if parser.encoding != yaml_ANY_ENCODING { + panic("must set the encoding only once") + } + parser.encoding = encoding +} + +// Create a new emitter object. +func yaml_emitter_initialize(emitter *yaml_emitter_t) { + *emitter = yaml_emitter_t{ + buffer: make([]byte, output_buffer_size), + raw_buffer: make([]byte, 0, output_raw_buffer_size), + states: make([]yaml_emitter_state_t, 0, initial_stack_size), + events: make([]yaml_event_t, 0, initial_queue_size), + } +} + +// Destroy an emitter object. +func yaml_emitter_delete(emitter *yaml_emitter_t) { + *emitter = yaml_emitter_t{} +} + +// String write handler. +func yaml_string_write_handler(emitter *yaml_emitter_t, buffer []byte) error { + *emitter.output_buffer = append(*emitter.output_buffer, buffer...) + return nil +} + +// yaml_writer_write_handler uses emitter.output_writer to write the +// emitted text. +func yaml_writer_write_handler(emitter *yaml_emitter_t, buffer []byte) error { + _, err := emitter.output_writer.Write(buffer) + return err +} + +// Set a string output. +func yaml_emitter_set_output_string(emitter *yaml_emitter_t, output_buffer *[]byte) { + if emitter.write_handler != nil { + panic("must set the output target only once") + } + emitter.write_handler = yaml_string_write_handler + emitter.output_buffer = output_buffer +} + +// Set a file output. +func yaml_emitter_set_output_writer(emitter *yaml_emitter_t, w io.Writer) { + if emitter.write_handler != nil { + panic("must set the output target only once") + } + emitter.write_handler = yaml_writer_write_handler + emitter.output_writer = w +} + +// Set the output encoding. +func yaml_emitter_set_encoding(emitter *yaml_emitter_t, encoding yaml_encoding_t) { + if emitter.encoding != yaml_ANY_ENCODING { + panic("must set the output encoding only once") + } + emitter.encoding = encoding +} + +// Set the canonical output style. +func yaml_emitter_set_canonical(emitter *yaml_emitter_t, canonical bool) { + emitter.canonical = canonical +} + +//// Set the indentation increment. +func yaml_emitter_set_indent(emitter *yaml_emitter_t, indent int) { + if indent < 2 || indent > 9 { + indent = 2 + } + emitter.best_indent = indent +} + +// Set the preferred line width. +func yaml_emitter_set_width(emitter *yaml_emitter_t, width int) { + if width < 0 { + width = -1 + } + emitter.best_width = width +} + +// Set if unescaped non-ASCII characters are allowed. +func yaml_emitter_set_unicode(emitter *yaml_emitter_t, unicode bool) { + emitter.unicode = unicode +} + +// Set the preferred line break character. +func yaml_emitter_set_break(emitter *yaml_emitter_t, line_break yaml_break_t) { + emitter.line_break = line_break +} + +///* +// * Destroy a token object. +// */ +// +//YAML_DECLARE(void) +//yaml_token_delete(yaml_token_t *token) +//{ +// assert(token); // Non-NULL token object expected. +// +// switch (token.type) +// { +// case YAML_TAG_DIRECTIVE_TOKEN: +// yaml_free(token.data.tag_directive.handle); +// yaml_free(token.data.tag_directive.prefix); +// break; +// +// case YAML_ALIAS_TOKEN: +// yaml_free(token.data.alias.value); +// break; +// +// case YAML_ANCHOR_TOKEN: +// yaml_free(token.data.anchor.value); +// break; +// +// case YAML_TAG_TOKEN: +// yaml_free(token.data.tag.handle); +// yaml_free(token.data.tag.suffix); +// break; +// +// case YAML_SCALAR_TOKEN: +// yaml_free(token.data.scalar.value); +// break; +// +// default: +// break; +// } +// +// memset(token, 0, sizeof(yaml_token_t)); +//} +// +///* +// * Check if a string is a valid UTF-8 sequence. +// * +// * Check 'reader.c' for more details on UTF-8 encoding. +// */ +// +//static int +//yaml_check_utf8(yaml_char_t *start, size_t length) +//{ +// yaml_char_t *end = start+length; +// yaml_char_t *pointer = start; +// +// while (pointer < end) { +// unsigned char octet; +// unsigned int width; +// unsigned int value; +// size_t k; +// +// octet = pointer[0]; +// width = (octet & 0x80) == 0x00 ? 1 : +// (octet & 0xE0) == 0xC0 ? 2 : +// (octet & 0xF0) == 0xE0 ? 3 : +// (octet & 0xF8) == 0xF0 ? 4 : 0; +// value = (octet & 0x80) == 0x00 ? octet & 0x7F : +// (octet & 0xE0) == 0xC0 ? octet & 0x1F : +// (octet & 0xF0) == 0xE0 ? octet & 0x0F : +// (octet & 0xF8) == 0xF0 ? octet & 0x07 : 0; +// if (!width) return 0; +// if (pointer+width > end) return 0; +// for (k = 1; k < width; k ++) { +// octet = pointer[k]; +// if ((octet & 0xC0) != 0x80) return 0; +// value = (value << 6) + (octet & 0x3F); +// } +// if (!((width == 1) || +// (width == 2 && value >= 0x80) || +// (width == 3 && value >= 0x800) || +// (width == 4 && value >= 0x10000))) return 0; +// +// pointer += width; +// } +// +// return 1; +//} +// + +// Create STREAM-START. +func yaml_stream_start_event_initialize(event *yaml_event_t, encoding yaml_encoding_t) { + *event = yaml_event_t{ + typ: yaml_STREAM_START_EVENT, + encoding: encoding, + } +} + +// Create STREAM-END. +func yaml_stream_end_event_initialize(event *yaml_event_t) { + *event = yaml_event_t{ + typ: yaml_STREAM_END_EVENT, + } +} + +// Create DOCUMENT-START. +func yaml_document_start_event_initialize( + event *yaml_event_t, + version_directive *yaml_version_directive_t, + tag_directives []yaml_tag_directive_t, + implicit bool, +) { + *event = yaml_event_t{ + typ: yaml_DOCUMENT_START_EVENT, + version_directive: version_directive, + tag_directives: tag_directives, + implicit: implicit, + } +} + +// Create DOCUMENT-END. +func yaml_document_end_event_initialize(event *yaml_event_t, implicit bool) { + *event = yaml_event_t{ + typ: yaml_DOCUMENT_END_EVENT, + implicit: implicit, + } +} + +///* +// * Create ALIAS. +// */ +// +//YAML_DECLARE(int) +//yaml_alias_event_initialize(event *yaml_event_t, anchor *yaml_char_t) +//{ +// mark yaml_mark_t = { 0, 0, 0 } +// anchor_copy *yaml_char_t = NULL +// +// assert(event) // Non-NULL event object is expected. +// assert(anchor) // Non-NULL anchor is expected. +// +// if (!yaml_check_utf8(anchor, strlen((char *)anchor))) return 0 +// +// anchor_copy = yaml_strdup(anchor) +// if (!anchor_copy) +// return 0 +// +// ALIAS_EVENT_INIT(*event, anchor_copy, mark, mark) +// +// return 1 +//} + +// Create SCALAR. +func yaml_scalar_event_initialize(event *yaml_event_t, anchor, tag, value []byte, plain_implicit, quoted_implicit bool, style yaml_scalar_style_t) bool { + *event = yaml_event_t{ + typ: yaml_SCALAR_EVENT, + anchor: anchor, + tag: tag, + value: value, + implicit: plain_implicit, + quoted_implicit: quoted_implicit, + style: yaml_style_t(style), + } + return true +} + +// Create SEQUENCE-START. +func yaml_sequence_start_event_initialize(event *yaml_event_t, anchor, tag []byte, implicit bool, style yaml_sequence_style_t) bool { + *event = yaml_event_t{ + typ: yaml_SEQUENCE_START_EVENT, + anchor: anchor, + tag: tag, + implicit: implicit, + style: yaml_style_t(style), + } + return true +} + +// Create SEQUENCE-END. +func yaml_sequence_end_event_initialize(event *yaml_event_t) bool { + *event = yaml_event_t{ + typ: yaml_SEQUENCE_END_EVENT, + } + return true +} + +// Create MAPPING-START. +func yaml_mapping_start_event_initialize(event *yaml_event_t, anchor, tag []byte, implicit bool, style yaml_mapping_style_t) { + *event = yaml_event_t{ + typ: yaml_MAPPING_START_EVENT, + anchor: anchor, + tag: tag, + implicit: implicit, + style: yaml_style_t(style), + } +} + +// Create MAPPING-END. +func yaml_mapping_end_event_initialize(event *yaml_event_t) { + *event = yaml_event_t{ + typ: yaml_MAPPING_END_EVENT, + } +} + +// Destroy an event object. +func yaml_event_delete(event *yaml_event_t) { + *event = yaml_event_t{} +} + +///* +// * Create a document object. +// */ +// +//YAML_DECLARE(int) +//yaml_document_initialize(document *yaml_document_t, +// version_directive *yaml_version_directive_t, +// tag_directives_start *yaml_tag_directive_t, +// tag_directives_end *yaml_tag_directive_t, +// start_implicit int, end_implicit int) +//{ +// struct { +// error yaml_error_type_t +// } context +// struct { +// start *yaml_node_t +// end *yaml_node_t +// top *yaml_node_t +// } nodes = { NULL, NULL, NULL } +// version_directive_copy *yaml_version_directive_t = NULL +// struct { +// start *yaml_tag_directive_t +// end *yaml_tag_directive_t +// top *yaml_tag_directive_t +// } tag_directives_copy = { NULL, NULL, NULL } +// value yaml_tag_directive_t = { NULL, NULL } +// mark yaml_mark_t = { 0, 0, 0 } +// +// assert(document) // Non-NULL document object is expected. +// assert((tag_directives_start && tag_directives_end) || +// (tag_directives_start == tag_directives_end)) +// // Valid tag directives are expected. +// +// if (!STACK_INIT(&context, nodes, INITIAL_STACK_SIZE)) goto error +// +// if (version_directive) { +// version_directive_copy = yaml_malloc(sizeof(yaml_version_directive_t)) +// if (!version_directive_copy) goto error +// version_directive_copy.major = version_directive.major +// version_directive_copy.minor = version_directive.minor +// } +// +// if (tag_directives_start != tag_directives_end) { +// tag_directive *yaml_tag_directive_t +// if (!STACK_INIT(&context, tag_directives_copy, INITIAL_STACK_SIZE)) +// goto error +// for (tag_directive = tag_directives_start +// tag_directive != tag_directives_end; tag_directive ++) { +// assert(tag_directive.handle) +// assert(tag_directive.prefix) +// if (!yaml_check_utf8(tag_directive.handle, +// strlen((char *)tag_directive.handle))) +// goto error +// if (!yaml_check_utf8(tag_directive.prefix, +// strlen((char *)tag_directive.prefix))) +// goto error +// value.handle = yaml_strdup(tag_directive.handle) +// value.prefix = yaml_strdup(tag_directive.prefix) +// if (!value.handle || !value.prefix) goto error +// if (!PUSH(&context, tag_directives_copy, value)) +// goto error +// value.handle = NULL +// value.prefix = NULL +// } +// } +// +// DOCUMENT_INIT(*document, nodes.start, nodes.end, version_directive_copy, +// tag_directives_copy.start, tag_directives_copy.top, +// start_implicit, end_implicit, mark, mark) +// +// return 1 +// +//error: +// STACK_DEL(&context, nodes) +// yaml_free(version_directive_copy) +// while (!STACK_EMPTY(&context, tag_directives_copy)) { +// value yaml_tag_directive_t = POP(&context, tag_directives_copy) +// yaml_free(value.handle) +// yaml_free(value.prefix) +// } +// STACK_DEL(&context, tag_directives_copy) +// yaml_free(value.handle) +// yaml_free(value.prefix) +// +// return 0 +//} +// +///* +// * Destroy a document object. +// */ +// +//YAML_DECLARE(void) +//yaml_document_delete(document *yaml_document_t) +//{ +// struct { +// error yaml_error_type_t +// } context +// tag_directive *yaml_tag_directive_t +// +// context.error = YAML_NO_ERROR // Eliminate a compiler warning. +// +// assert(document) // Non-NULL document object is expected. +// +// while (!STACK_EMPTY(&context, document.nodes)) { +// node yaml_node_t = POP(&context, document.nodes) +// yaml_free(node.tag) +// switch (node.type) { +// case YAML_SCALAR_NODE: +// yaml_free(node.data.scalar.value) +// break +// case YAML_SEQUENCE_NODE: +// STACK_DEL(&context, node.data.sequence.items) +// break +// case YAML_MAPPING_NODE: +// STACK_DEL(&context, node.data.mapping.pairs) +// break +// default: +// assert(0) // Should not happen. +// } +// } +// STACK_DEL(&context, document.nodes) +// +// yaml_free(document.version_directive) +// for (tag_directive = document.tag_directives.start +// tag_directive != document.tag_directives.end +// tag_directive++) { +// yaml_free(tag_directive.handle) +// yaml_free(tag_directive.prefix) +// } +// yaml_free(document.tag_directives.start) +// +// memset(document, 0, sizeof(yaml_document_t)) +//} +// +///** +// * Get a document node. +// */ +// +//YAML_DECLARE(yaml_node_t *) +//yaml_document_get_node(document *yaml_document_t, index int) +//{ +// assert(document) // Non-NULL document object is expected. +// +// if (index > 0 && document.nodes.start + index <= document.nodes.top) { +// return document.nodes.start + index - 1 +// } +// return NULL +//} +// +///** +// * Get the root object. +// */ +// +//YAML_DECLARE(yaml_node_t *) +//yaml_document_get_root_node(document *yaml_document_t) +//{ +// assert(document) // Non-NULL document object is expected. +// +// if (document.nodes.top != document.nodes.start) { +// return document.nodes.start +// } +// return NULL +//} +// +///* +// * Add a scalar node to a document. +// */ +// +//YAML_DECLARE(int) +//yaml_document_add_scalar(document *yaml_document_t, +// tag *yaml_char_t, value *yaml_char_t, length int, +// style yaml_scalar_style_t) +//{ +// struct { +// error yaml_error_type_t +// } context +// mark yaml_mark_t = { 0, 0, 0 } +// tag_copy *yaml_char_t = NULL +// value_copy *yaml_char_t = NULL +// node yaml_node_t +// +// assert(document) // Non-NULL document object is expected. +// assert(value) // Non-NULL value is expected. +// +// if (!tag) { +// tag = (yaml_char_t *)YAML_DEFAULT_SCALAR_TAG +// } +// +// if (!yaml_check_utf8(tag, strlen((char *)tag))) goto error +// tag_copy = yaml_strdup(tag) +// if (!tag_copy) goto error +// +// if (length < 0) { +// length = strlen((char *)value) +// } +// +// if (!yaml_check_utf8(value, length)) goto error +// value_copy = yaml_malloc(length+1) +// if (!value_copy) goto error +// memcpy(value_copy, value, length) +// value_copy[length] = '\0' +// +// SCALAR_NODE_INIT(node, tag_copy, value_copy, length, style, mark, mark) +// if (!PUSH(&context, document.nodes, node)) goto error +// +// return document.nodes.top - document.nodes.start +// +//error: +// yaml_free(tag_copy) +// yaml_free(value_copy) +// +// return 0 +//} +// +///* +// * Add a sequence node to a document. +// */ +// +//YAML_DECLARE(int) +//yaml_document_add_sequence(document *yaml_document_t, +// tag *yaml_char_t, style yaml_sequence_style_t) +//{ +// struct { +// error yaml_error_type_t +// } context +// mark yaml_mark_t = { 0, 0, 0 } +// tag_copy *yaml_char_t = NULL +// struct { +// start *yaml_node_item_t +// end *yaml_node_item_t +// top *yaml_node_item_t +// } items = { NULL, NULL, NULL } +// node yaml_node_t +// +// assert(document) // Non-NULL document object is expected. +// +// if (!tag) { +// tag = (yaml_char_t *)YAML_DEFAULT_SEQUENCE_TAG +// } +// +// if (!yaml_check_utf8(tag, strlen((char *)tag))) goto error +// tag_copy = yaml_strdup(tag) +// if (!tag_copy) goto error +// +// if (!STACK_INIT(&context, items, INITIAL_STACK_SIZE)) goto error +// +// SEQUENCE_NODE_INIT(node, tag_copy, items.start, items.end, +// style, mark, mark) +// if (!PUSH(&context, document.nodes, node)) goto error +// +// return document.nodes.top - document.nodes.start +// +//error: +// STACK_DEL(&context, items) +// yaml_free(tag_copy) +// +// return 0 +//} +// +///* +// * Add a mapping node to a document. +// */ +// +//YAML_DECLARE(int) +//yaml_document_add_mapping(document *yaml_document_t, +// tag *yaml_char_t, style yaml_mapping_style_t) +//{ +// struct { +// error yaml_error_type_t +// } context +// mark yaml_mark_t = { 0, 0, 0 } +// tag_copy *yaml_char_t = NULL +// struct { +// start *yaml_node_pair_t +// end *yaml_node_pair_t +// top *yaml_node_pair_t +// } pairs = { NULL, NULL, NULL } +// node yaml_node_t +// +// assert(document) // Non-NULL document object is expected. +// +// if (!tag) { +// tag = (yaml_char_t *)YAML_DEFAULT_MAPPING_TAG +// } +// +// if (!yaml_check_utf8(tag, strlen((char *)tag))) goto error +// tag_copy = yaml_strdup(tag) +// if (!tag_copy) goto error +// +// if (!STACK_INIT(&context, pairs, INITIAL_STACK_SIZE)) goto error +// +// MAPPING_NODE_INIT(node, tag_copy, pairs.start, pairs.end, +// style, mark, mark) +// if (!PUSH(&context, document.nodes, node)) goto error +// +// return document.nodes.top - document.nodes.start +// +//error: +// STACK_DEL(&context, pairs) +// yaml_free(tag_copy) +// +// return 0 +//} +// +///* +// * Append an item to a sequence node. +// */ +// +//YAML_DECLARE(int) +//yaml_document_append_sequence_item(document *yaml_document_t, +// sequence int, item int) +//{ +// struct { +// error yaml_error_type_t +// } context +// +// assert(document) // Non-NULL document is required. +// assert(sequence > 0 +// && document.nodes.start + sequence <= document.nodes.top) +// // Valid sequence id is required. +// assert(document.nodes.start[sequence-1].type == YAML_SEQUENCE_NODE) +// // A sequence node is required. +// assert(item > 0 && document.nodes.start + item <= document.nodes.top) +// // Valid item id is required. +// +// if (!PUSH(&context, +// document.nodes.start[sequence-1].data.sequence.items, item)) +// return 0 +// +// return 1 +//} +// +///* +// * Append a pair of a key and a value to a mapping node. +// */ +// +//YAML_DECLARE(int) +//yaml_document_append_mapping_pair(document *yaml_document_t, +// mapping int, key int, value int) +//{ +// struct { +// error yaml_error_type_t +// } context +// +// pair yaml_node_pair_t +// +// assert(document) // Non-NULL document is required. +// assert(mapping > 0 +// && document.nodes.start + mapping <= document.nodes.top) +// // Valid mapping id is required. +// assert(document.nodes.start[mapping-1].type == YAML_MAPPING_NODE) +// // A mapping node is required. +// assert(key > 0 && document.nodes.start + key <= document.nodes.top) +// // Valid key id is required. +// assert(value > 0 && document.nodes.start + value <= document.nodes.top) +// // Valid value id is required. +// +// pair.key = key +// pair.value = value +// +// if (!PUSH(&context, +// document.nodes.start[mapping-1].data.mapping.pairs, pair)) +// return 0 +// +// return 1 +//} +// +// diff --git a/tools/voluspa/vendor/gopkg.in/yaml.v2/decode.go b/tools/voluspa/vendor/gopkg.in/yaml.v2/decode.go new file mode 100644 index 00000000000..e4e56e28e0e --- /dev/null +++ b/tools/voluspa/vendor/gopkg.in/yaml.v2/decode.go @@ -0,0 +1,775 @@ +package yaml + +import ( + "encoding" + "encoding/base64" + "fmt" + "io" + "math" + "reflect" + "strconv" + "time" +) + +const ( + documentNode = 1 << iota + mappingNode + sequenceNode + scalarNode + aliasNode +) + +type node struct { + kind int + line, column int + tag string + // For an alias node, alias holds the resolved alias. + alias *node + value string + implicit bool + children []*node + anchors map[string]*node +} + +// ---------------------------------------------------------------------------- +// Parser, produces a node tree out of a libyaml event stream. + +type parser struct { + parser yaml_parser_t + event yaml_event_t + doc *node + doneInit bool +} + +func newParser(b []byte) *parser { + p := parser{} + if !yaml_parser_initialize(&p.parser) { + panic("failed to initialize YAML emitter") + } + if len(b) == 0 { + b = []byte{'\n'} + } + yaml_parser_set_input_string(&p.parser, b) + return &p +} + +func newParserFromReader(r io.Reader) *parser { + p := parser{} + if !yaml_parser_initialize(&p.parser) { + panic("failed to initialize YAML emitter") + } + yaml_parser_set_input_reader(&p.parser, r) + return &p +} + +func (p *parser) init() { + if p.doneInit { + return + } + p.expect(yaml_STREAM_START_EVENT) + p.doneInit = true +} + +func (p *parser) destroy() { + if p.event.typ != yaml_NO_EVENT { + yaml_event_delete(&p.event) + } + yaml_parser_delete(&p.parser) +} + +// expect consumes an event from the event stream and +// checks that it's of the expected type. +func (p *parser) expect(e yaml_event_type_t) { + if p.event.typ == yaml_NO_EVENT { + if !yaml_parser_parse(&p.parser, &p.event) { + p.fail() + } + } + if p.event.typ == yaml_STREAM_END_EVENT { + failf("attempted to go past the end of stream; corrupted value?") + } + if p.event.typ != e { + p.parser.problem = fmt.Sprintf("expected %s event but got %s", e, p.event.typ) + p.fail() + } + yaml_event_delete(&p.event) + p.event.typ = yaml_NO_EVENT +} + +// peek peeks at the next event in the event stream, +// puts the results into p.event and returns the event type. +func (p *parser) peek() yaml_event_type_t { + if p.event.typ != yaml_NO_EVENT { + return p.event.typ + } + if !yaml_parser_parse(&p.parser, &p.event) { + p.fail() + } + return p.event.typ +} + +func (p *parser) fail() { + var where string + var line int + if p.parser.problem_mark.line != 0 { + line = p.parser.problem_mark.line + // Scanner errors don't iterate line before returning error + if p.parser.error == yaml_SCANNER_ERROR { + line++ + } + } else if p.parser.context_mark.line != 0 { + line = p.parser.context_mark.line + } + if line != 0 { + where = "line " + strconv.Itoa(line) + ": " + } + var msg string + if len(p.parser.problem) > 0 { + msg = p.parser.problem + } else { + msg = "unknown problem parsing YAML content" + } + failf("%s%s", where, msg) +} + +func (p *parser) anchor(n *node, anchor []byte) { + if anchor != nil { + p.doc.anchors[string(anchor)] = n + } +} + +func (p *parser) parse() *node { + p.init() + switch p.peek() { + case yaml_SCALAR_EVENT: + return p.scalar() + case yaml_ALIAS_EVENT: + return p.alias() + case yaml_MAPPING_START_EVENT: + return p.mapping() + case yaml_SEQUENCE_START_EVENT: + return p.sequence() + case yaml_DOCUMENT_START_EVENT: + return p.document() + case yaml_STREAM_END_EVENT: + // Happens when attempting to decode an empty buffer. + return nil + default: + panic("attempted to parse unknown event: " + p.event.typ.String()) + } +} + +func (p *parser) node(kind int) *node { + return &node{ + kind: kind, + line: p.event.start_mark.line, + column: p.event.start_mark.column, + } +} + +func (p *parser) document() *node { + n := p.node(documentNode) + n.anchors = make(map[string]*node) + p.doc = n + p.expect(yaml_DOCUMENT_START_EVENT) + n.children = append(n.children, p.parse()) + p.expect(yaml_DOCUMENT_END_EVENT) + return n +} + +func (p *parser) alias() *node { + n := p.node(aliasNode) + n.value = string(p.event.anchor) + n.alias = p.doc.anchors[n.value] + if n.alias == nil { + failf("unknown anchor '%s' referenced", n.value) + } + p.expect(yaml_ALIAS_EVENT) + return n +} + +func (p *parser) scalar() *node { + n := p.node(scalarNode) + n.value = string(p.event.value) + n.tag = string(p.event.tag) + n.implicit = p.event.implicit + p.anchor(n, p.event.anchor) + p.expect(yaml_SCALAR_EVENT) + return n +} + +func (p *parser) sequence() *node { + n := p.node(sequenceNode) + p.anchor(n, p.event.anchor) + p.expect(yaml_SEQUENCE_START_EVENT) + for p.peek() != yaml_SEQUENCE_END_EVENT { + n.children = append(n.children, p.parse()) + } + p.expect(yaml_SEQUENCE_END_EVENT) + return n +} + +func (p *parser) mapping() *node { + n := p.node(mappingNode) + p.anchor(n, p.event.anchor) + p.expect(yaml_MAPPING_START_EVENT) + for p.peek() != yaml_MAPPING_END_EVENT { + n.children = append(n.children, p.parse(), p.parse()) + } + p.expect(yaml_MAPPING_END_EVENT) + return n +} + +// ---------------------------------------------------------------------------- +// Decoder, unmarshals a node into a provided value. + +type decoder struct { + doc *node + aliases map[*node]bool + mapType reflect.Type + terrors []string + strict bool +} + +var ( + mapItemType = reflect.TypeOf(MapItem{}) + durationType = reflect.TypeOf(time.Duration(0)) + defaultMapType = reflect.TypeOf(map[interface{}]interface{}{}) + ifaceType = defaultMapType.Elem() + timeType = reflect.TypeOf(time.Time{}) + ptrTimeType = reflect.TypeOf(&time.Time{}) +) + +func newDecoder(strict bool) *decoder { + d := &decoder{mapType: defaultMapType, strict: strict} + d.aliases = make(map[*node]bool) + return d +} + +func (d *decoder) terror(n *node, tag string, out reflect.Value) { + if n.tag != "" { + tag = n.tag + } + value := n.value + if tag != yaml_SEQ_TAG && tag != yaml_MAP_TAG { + if len(value) > 10 { + value = " `" + value[:7] + "...`" + } else { + value = " `" + value + "`" + } + } + d.terrors = append(d.terrors, fmt.Sprintf("line %d: cannot unmarshal %s%s into %s", n.line+1, shortTag(tag), value, out.Type())) +} + +func (d *decoder) callUnmarshaler(n *node, u Unmarshaler) (good bool) { + terrlen := len(d.terrors) + err := u.UnmarshalYAML(func(v interface{}) (err error) { + defer handleErr(&err) + d.unmarshal(n, reflect.ValueOf(v)) + if len(d.terrors) > terrlen { + issues := d.terrors[terrlen:] + d.terrors = d.terrors[:terrlen] + return &TypeError{issues} + } + return nil + }) + if e, ok := err.(*TypeError); ok { + d.terrors = append(d.terrors, e.Errors...) + return false + } + if err != nil { + fail(err) + } + return true +} + +// d.prepare initializes and dereferences pointers and calls UnmarshalYAML +// if a value is found to implement it. +// It returns the initialized and dereferenced out value, whether +// unmarshalling was already done by UnmarshalYAML, and if so whether +// its types unmarshalled appropriately. +// +// If n holds a null value, prepare returns before doing anything. +func (d *decoder) prepare(n *node, out reflect.Value) (newout reflect.Value, unmarshaled, good bool) { + if n.tag == yaml_NULL_TAG || n.kind == scalarNode && n.tag == "" && (n.value == "null" || n.value == "~" || n.value == "" && n.implicit) { + return out, false, false + } + again := true + for again { + again = false + if out.Kind() == reflect.Ptr { + if out.IsNil() { + out.Set(reflect.New(out.Type().Elem())) + } + out = out.Elem() + again = true + } + if out.CanAddr() { + if u, ok := out.Addr().Interface().(Unmarshaler); ok { + good = d.callUnmarshaler(n, u) + return out, true, good + } + } + } + return out, false, false +} + +func (d *decoder) unmarshal(n *node, out reflect.Value) (good bool) { + switch n.kind { + case documentNode: + return d.document(n, out) + case aliasNode: + return d.alias(n, out) + } + out, unmarshaled, good := d.prepare(n, out) + if unmarshaled { + return good + } + switch n.kind { + case scalarNode: + good = d.scalar(n, out) + case mappingNode: + good = d.mapping(n, out) + case sequenceNode: + good = d.sequence(n, out) + default: + panic("internal error: unknown node kind: " + strconv.Itoa(n.kind)) + } + return good +} + +func (d *decoder) document(n *node, out reflect.Value) (good bool) { + if len(n.children) == 1 { + d.doc = n + d.unmarshal(n.children[0], out) + return true + } + return false +} + +func (d *decoder) alias(n *node, out reflect.Value) (good bool) { + if d.aliases[n] { + // TODO this could actually be allowed in some circumstances. + failf("anchor '%s' value contains itself", n.value) + } + d.aliases[n] = true + good = d.unmarshal(n.alias, out) + delete(d.aliases, n) + return good +} + +var zeroValue reflect.Value + +func resetMap(out reflect.Value) { + for _, k := range out.MapKeys() { + out.SetMapIndex(k, zeroValue) + } +} + +func (d *decoder) scalar(n *node, out reflect.Value) bool { + var tag string + var resolved interface{} + if n.tag == "" && !n.implicit { + tag = yaml_STR_TAG + resolved = n.value + } else { + tag, resolved = resolve(n.tag, n.value) + if tag == yaml_BINARY_TAG { + data, err := base64.StdEncoding.DecodeString(resolved.(string)) + if err != nil { + failf("!!binary value contains invalid base64 data") + } + resolved = string(data) + } + } + if resolved == nil { + if out.Kind() == reflect.Map && !out.CanAddr() { + resetMap(out) + } else { + out.Set(reflect.Zero(out.Type())) + } + return true + } + if resolvedv := reflect.ValueOf(resolved); out.Type() == resolvedv.Type() { + // We've resolved to exactly the type we want, so use that. + out.Set(resolvedv) + return true + } + // Perhaps we can use the value as a TextUnmarshaler to + // set its value. + if out.CanAddr() { + u, ok := out.Addr().Interface().(encoding.TextUnmarshaler) + if ok { + var text []byte + if tag == yaml_BINARY_TAG { + text = []byte(resolved.(string)) + } else { + // We let any value be unmarshaled into TextUnmarshaler. + // That might be more lax than we'd like, but the + // TextUnmarshaler itself should bowl out any dubious values. + text = []byte(n.value) + } + err := u.UnmarshalText(text) + if err != nil { + fail(err) + } + return true + } + } + switch out.Kind() { + case reflect.String: + if tag == yaml_BINARY_TAG { + out.SetString(resolved.(string)) + return true + } + if resolved != nil { + out.SetString(n.value) + return true + } + case reflect.Interface: + if resolved == nil { + out.Set(reflect.Zero(out.Type())) + } else if tag == yaml_TIMESTAMP_TAG { + // It looks like a timestamp but for backward compatibility + // reasons we set it as a string, so that code that unmarshals + // timestamp-like values into interface{} will continue to + // see a string and not a time.Time. + // TODO(v3) Drop this. + out.Set(reflect.ValueOf(n.value)) + } else { + out.Set(reflect.ValueOf(resolved)) + } + return true + case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64: + switch resolved := resolved.(type) { + case int: + if !out.OverflowInt(int64(resolved)) { + out.SetInt(int64(resolved)) + return true + } + case int64: + if !out.OverflowInt(resolved) { + out.SetInt(resolved) + return true + } + case uint64: + if resolved <= math.MaxInt64 && !out.OverflowInt(int64(resolved)) { + out.SetInt(int64(resolved)) + return true + } + case float64: + if resolved <= math.MaxInt64 && !out.OverflowInt(int64(resolved)) { + out.SetInt(int64(resolved)) + return true + } + case string: + if out.Type() == durationType { + d, err := time.ParseDuration(resolved) + if err == nil { + out.SetInt(int64(d)) + return true + } + } + } + case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64, reflect.Uintptr: + switch resolved := resolved.(type) { + case int: + if resolved >= 0 && !out.OverflowUint(uint64(resolved)) { + out.SetUint(uint64(resolved)) + return true + } + case int64: + if resolved >= 0 && !out.OverflowUint(uint64(resolved)) { + out.SetUint(uint64(resolved)) + return true + } + case uint64: + if !out.OverflowUint(uint64(resolved)) { + out.SetUint(uint64(resolved)) + return true + } + case float64: + if resolved <= math.MaxUint64 && !out.OverflowUint(uint64(resolved)) { + out.SetUint(uint64(resolved)) + return true + } + } + case reflect.Bool: + switch resolved := resolved.(type) { + case bool: + out.SetBool(resolved) + return true + } + case reflect.Float32, reflect.Float64: + switch resolved := resolved.(type) { + case int: + out.SetFloat(float64(resolved)) + return true + case int64: + out.SetFloat(float64(resolved)) + return true + case uint64: + out.SetFloat(float64(resolved)) + return true + case float64: + out.SetFloat(resolved) + return true + } + case reflect.Struct: + if resolvedv := reflect.ValueOf(resolved); out.Type() == resolvedv.Type() { + out.Set(resolvedv) + return true + } + case reflect.Ptr: + if out.Type().Elem() == reflect.TypeOf(resolved) { + // TODO DOes this make sense? When is out a Ptr except when decoding a nil value? + elem := reflect.New(out.Type().Elem()) + elem.Elem().Set(reflect.ValueOf(resolved)) + out.Set(elem) + return true + } + } + d.terror(n, tag, out) + return false +} + +func settableValueOf(i interface{}) reflect.Value { + v := reflect.ValueOf(i) + sv := reflect.New(v.Type()).Elem() + sv.Set(v) + return sv +} + +func (d *decoder) sequence(n *node, out reflect.Value) (good bool) { + l := len(n.children) + + var iface reflect.Value + switch out.Kind() { + case reflect.Slice: + out.Set(reflect.MakeSlice(out.Type(), l, l)) + case reflect.Array: + if l != out.Len() { + failf("invalid array: want %d elements but got %d", out.Len(), l) + } + case reflect.Interface: + // No type hints. Will have to use a generic sequence. + iface = out + out = settableValueOf(make([]interface{}, l)) + default: + d.terror(n, yaml_SEQ_TAG, out) + return false + } + et := out.Type().Elem() + + j := 0 + for i := 0; i < l; i++ { + e := reflect.New(et).Elem() + if ok := d.unmarshal(n.children[i], e); ok { + out.Index(j).Set(e) + j++ + } + } + if out.Kind() != reflect.Array { + out.Set(out.Slice(0, j)) + } + if iface.IsValid() { + iface.Set(out) + } + return true +} + +func (d *decoder) mapping(n *node, out reflect.Value) (good bool) { + switch out.Kind() { + case reflect.Struct: + return d.mappingStruct(n, out) + case reflect.Slice: + return d.mappingSlice(n, out) + case reflect.Map: + // okay + case reflect.Interface: + if d.mapType.Kind() == reflect.Map { + iface := out + out = reflect.MakeMap(d.mapType) + iface.Set(out) + } else { + slicev := reflect.New(d.mapType).Elem() + if !d.mappingSlice(n, slicev) { + return false + } + out.Set(slicev) + return true + } + default: + d.terror(n, yaml_MAP_TAG, out) + return false + } + outt := out.Type() + kt := outt.Key() + et := outt.Elem() + + mapType := d.mapType + if outt.Key() == ifaceType && outt.Elem() == ifaceType { + d.mapType = outt + } + + if out.IsNil() { + out.Set(reflect.MakeMap(outt)) + } + l := len(n.children) + for i := 0; i < l; i += 2 { + if isMerge(n.children[i]) { + d.merge(n.children[i+1], out) + continue + } + k := reflect.New(kt).Elem() + if d.unmarshal(n.children[i], k) { + kkind := k.Kind() + if kkind == reflect.Interface { + kkind = k.Elem().Kind() + } + if kkind == reflect.Map || kkind == reflect.Slice { + failf("invalid map key: %#v", k.Interface()) + } + e := reflect.New(et).Elem() + if d.unmarshal(n.children[i+1], e) { + d.setMapIndex(n.children[i+1], out, k, e) + } + } + } + d.mapType = mapType + return true +} + +func (d *decoder) setMapIndex(n *node, out, k, v reflect.Value) { + if d.strict && out.MapIndex(k) != zeroValue { + d.terrors = append(d.terrors, fmt.Sprintf("line %d: key %#v already set in map", n.line+1, k.Interface())) + return + } + out.SetMapIndex(k, v) +} + +func (d *decoder) mappingSlice(n *node, out reflect.Value) (good bool) { + outt := out.Type() + if outt.Elem() != mapItemType { + d.terror(n, yaml_MAP_TAG, out) + return false + } + + mapType := d.mapType + d.mapType = outt + + var slice []MapItem + var l = len(n.children) + for i := 0; i < l; i += 2 { + if isMerge(n.children[i]) { + d.merge(n.children[i+1], out) + continue + } + item := MapItem{} + k := reflect.ValueOf(&item.Key).Elem() + if d.unmarshal(n.children[i], k) { + v := reflect.ValueOf(&item.Value).Elem() + if d.unmarshal(n.children[i+1], v) { + slice = append(slice, item) + } + } + } + out.Set(reflect.ValueOf(slice)) + d.mapType = mapType + return true +} + +func (d *decoder) mappingStruct(n *node, out reflect.Value) (good bool) { + sinfo, err := getStructInfo(out.Type()) + if err != nil { + panic(err) + } + name := settableValueOf("") + l := len(n.children) + + var inlineMap reflect.Value + var elemType reflect.Type + if sinfo.InlineMap != -1 { + inlineMap = out.Field(sinfo.InlineMap) + inlineMap.Set(reflect.New(inlineMap.Type()).Elem()) + elemType = inlineMap.Type().Elem() + } + + var doneFields []bool + if d.strict { + doneFields = make([]bool, len(sinfo.FieldsList)) + } + for i := 0; i < l; i += 2 { + ni := n.children[i] + if isMerge(ni) { + d.merge(n.children[i+1], out) + continue + } + if !d.unmarshal(ni, name) { + continue + } + if info, ok := sinfo.FieldsMap[name.String()]; ok { + if d.strict { + if doneFields[info.Id] { + d.terrors = append(d.terrors, fmt.Sprintf("line %d: field %s already set in type %s", ni.line+1, name.String(), out.Type())) + continue + } + doneFields[info.Id] = true + } + var field reflect.Value + if info.Inline == nil { + field = out.Field(info.Num) + } else { + field = out.FieldByIndex(info.Inline) + } + d.unmarshal(n.children[i+1], field) + } else if sinfo.InlineMap != -1 { + if inlineMap.IsNil() { + inlineMap.Set(reflect.MakeMap(inlineMap.Type())) + } + value := reflect.New(elemType).Elem() + d.unmarshal(n.children[i+1], value) + d.setMapIndex(n.children[i+1], inlineMap, name, value) + } else if d.strict { + d.terrors = append(d.terrors, fmt.Sprintf("line %d: field %s not found in type %s", ni.line+1, name.String(), out.Type())) + } + } + return true +} + +func failWantMap() { + failf("map merge requires map or sequence of maps as the value") +} + +func (d *decoder) merge(n *node, out reflect.Value) { + switch n.kind { + case mappingNode: + d.unmarshal(n, out) + case aliasNode: + an, ok := d.doc.anchors[n.value] + if ok && an.kind != mappingNode { + failWantMap() + } + d.unmarshal(n, out) + case sequenceNode: + // Step backwards as earlier nodes take precedence. + for i := len(n.children) - 1; i >= 0; i-- { + ni := n.children[i] + if ni.kind == aliasNode { + an, ok := d.doc.anchors[ni.value] + if ok && an.kind != mappingNode { + failWantMap() + } + } else if ni.kind != mappingNode { + failWantMap() + } + d.unmarshal(ni, out) + } + default: + failWantMap() + } +} + +func isMerge(n *node) bool { + return n.kind == scalarNode && n.value == "<<" && (n.implicit == true || n.tag == yaml_MERGE_TAG) +} diff --git a/tools/voluspa/vendor/gopkg.in/yaml.v2/emitterc.go b/tools/voluspa/vendor/gopkg.in/yaml.v2/emitterc.go new file mode 100644 index 00000000000..a1c2cc52627 --- /dev/null +++ b/tools/voluspa/vendor/gopkg.in/yaml.v2/emitterc.go @@ -0,0 +1,1685 @@ +package yaml + +import ( + "bytes" + "fmt" +) + +// Flush the buffer if needed. +func flush(emitter *yaml_emitter_t) bool { + if emitter.buffer_pos+5 >= len(emitter.buffer) { + return yaml_emitter_flush(emitter) + } + return true +} + +// Put a character to the output buffer. +func put(emitter *yaml_emitter_t, value byte) bool { + if emitter.buffer_pos+5 >= len(emitter.buffer) && !yaml_emitter_flush(emitter) { + return false + } + emitter.buffer[emitter.buffer_pos] = value + emitter.buffer_pos++ + emitter.column++ + return true +} + +// Put a line break to the output buffer. +func put_break(emitter *yaml_emitter_t) bool { + if emitter.buffer_pos+5 >= len(emitter.buffer) && !yaml_emitter_flush(emitter) { + return false + } + switch emitter.line_break { + case yaml_CR_BREAK: + emitter.buffer[emitter.buffer_pos] = '\r' + emitter.buffer_pos += 1 + case yaml_LN_BREAK: + emitter.buffer[emitter.buffer_pos] = '\n' + emitter.buffer_pos += 1 + case yaml_CRLN_BREAK: + emitter.buffer[emitter.buffer_pos+0] = '\r' + emitter.buffer[emitter.buffer_pos+1] = '\n' + emitter.buffer_pos += 2 + default: + panic("unknown line break setting") + } + emitter.column = 0 + emitter.line++ + return true +} + +// Copy a character from a string into buffer. +func write(emitter *yaml_emitter_t, s []byte, i *int) bool { + if emitter.buffer_pos+5 >= len(emitter.buffer) && !yaml_emitter_flush(emitter) { + return false + } + p := emitter.buffer_pos + w := width(s[*i]) + switch w { + case 4: + emitter.buffer[p+3] = s[*i+3] + fallthrough + case 3: + emitter.buffer[p+2] = s[*i+2] + fallthrough + case 2: + emitter.buffer[p+1] = s[*i+1] + fallthrough + case 1: + emitter.buffer[p+0] = s[*i+0] + default: + panic("unknown character width") + } + emitter.column++ + emitter.buffer_pos += w + *i += w + return true +} + +// Write a whole string into buffer. +func write_all(emitter *yaml_emitter_t, s []byte) bool { + for i := 0; i < len(s); { + if !write(emitter, s, &i) { + return false + } + } + return true +} + +// Copy a line break character from a string into buffer. +func write_break(emitter *yaml_emitter_t, s []byte, i *int) bool { + if s[*i] == '\n' { + if !put_break(emitter) { + return false + } + *i++ + } else { + if !write(emitter, s, i) { + return false + } + emitter.column = 0 + emitter.line++ + } + return true +} + +// Set an emitter error and return false. +func yaml_emitter_set_emitter_error(emitter *yaml_emitter_t, problem string) bool { + emitter.error = yaml_EMITTER_ERROR + emitter.problem = problem + return false +} + +// Emit an event. +func yaml_emitter_emit(emitter *yaml_emitter_t, event *yaml_event_t) bool { + emitter.events = append(emitter.events, *event) + for !yaml_emitter_need_more_events(emitter) { + event := &emitter.events[emitter.events_head] + if !yaml_emitter_analyze_event(emitter, event) { + return false + } + if !yaml_emitter_state_machine(emitter, event) { + return false + } + yaml_event_delete(event) + emitter.events_head++ + } + return true +} + +// Check if we need to accumulate more events before emitting. +// +// We accumulate extra +// - 1 event for DOCUMENT-START +// - 2 events for SEQUENCE-START +// - 3 events for MAPPING-START +// +func yaml_emitter_need_more_events(emitter *yaml_emitter_t) bool { + if emitter.events_head == len(emitter.events) { + return true + } + var accumulate int + switch emitter.events[emitter.events_head].typ { + case yaml_DOCUMENT_START_EVENT: + accumulate = 1 + break + case yaml_SEQUENCE_START_EVENT: + accumulate = 2 + break + case yaml_MAPPING_START_EVENT: + accumulate = 3 + break + default: + return false + } + if len(emitter.events)-emitter.events_head > accumulate { + return false + } + var level int + for i := emitter.events_head; i < len(emitter.events); i++ { + switch emitter.events[i].typ { + case yaml_STREAM_START_EVENT, yaml_DOCUMENT_START_EVENT, yaml_SEQUENCE_START_EVENT, yaml_MAPPING_START_EVENT: + level++ + case yaml_STREAM_END_EVENT, yaml_DOCUMENT_END_EVENT, yaml_SEQUENCE_END_EVENT, yaml_MAPPING_END_EVENT: + level-- + } + if level == 0 { + return false + } + } + return true +} + +// Append a directive to the directives stack. +func yaml_emitter_append_tag_directive(emitter *yaml_emitter_t, value *yaml_tag_directive_t, allow_duplicates bool) bool { + for i := 0; i < len(emitter.tag_directives); i++ { + if bytes.Equal(value.handle, emitter.tag_directives[i].handle) { + if allow_duplicates { + return true + } + return yaml_emitter_set_emitter_error(emitter, "duplicate %TAG directive") + } + } + + // [Go] Do we actually need to copy this given garbage collection + // and the lack of deallocating destructors? + tag_copy := yaml_tag_directive_t{ + handle: make([]byte, len(value.handle)), + prefix: make([]byte, len(value.prefix)), + } + copy(tag_copy.handle, value.handle) + copy(tag_copy.prefix, value.prefix) + emitter.tag_directives = append(emitter.tag_directives, tag_copy) + return true +} + +// Increase the indentation level. +func yaml_emitter_increase_indent(emitter *yaml_emitter_t, flow, indentless bool) bool { + emitter.indents = append(emitter.indents, emitter.indent) + if emitter.indent < 0 { + if flow { + emitter.indent = emitter.best_indent + } else { + emitter.indent = 0 + } + } else if !indentless { + emitter.indent += emitter.best_indent + } + return true +} + +// State dispatcher. +func yaml_emitter_state_machine(emitter *yaml_emitter_t, event *yaml_event_t) bool { + switch emitter.state { + default: + case yaml_EMIT_STREAM_START_STATE: + return yaml_emitter_emit_stream_start(emitter, event) + + case yaml_EMIT_FIRST_DOCUMENT_START_STATE: + return yaml_emitter_emit_document_start(emitter, event, true) + + case yaml_EMIT_DOCUMENT_START_STATE: + return yaml_emitter_emit_document_start(emitter, event, false) + + case yaml_EMIT_DOCUMENT_CONTENT_STATE: + return yaml_emitter_emit_document_content(emitter, event) + + case yaml_EMIT_DOCUMENT_END_STATE: + return yaml_emitter_emit_document_end(emitter, event) + + case yaml_EMIT_FLOW_SEQUENCE_FIRST_ITEM_STATE: + return yaml_emitter_emit_flow_sequence_item(emitter, event, true) + + case yaml_EMIT_FLOW_SEQUENCE_ITEM_STATE: + return yaml_emitter_emit_flow_sequence_item(emitter, event, false) + + case yaml_EMIT_FLOW_MAPPING_FIRST_KEY_STATE: + return yaml_emitter_emit_flow_mapping_key(emitter, event, true) + + case yaml_EMIT_FLOW_MAPPING_KEY_STATE: + return yaml_emitter_emit_flow_mapping_key(emitter, event, false) + + case yaml_EMIT_FLOW_MAPPING_SIMPLE_VALUE_STATE: + return yaml_emitter_emit_flow_mapping_value(emitter, event, true) + + case yaml_EMIT_FLOW_MAPPING_VALUE_STATE: + return yaml_emitter_emit_flow_mapping_value(emitter, event, false) + + case yaml_EMIT_BLOCK_SEQUENCE_FIRST_ITEM_STATE: + return yaml_emitter_emit_block_sequence_item(emitter, event, true) + + case yaml_EMIT_BLOCK_SEQUENCE_ITEM_STATE: + return yaml_emitter_emit_block_sequence_item(emitter, event, false) + + case yaml_EMIT_BLOCK_MAPPING_FIRST_KEY_STATE: + return yaml_emitter_emit_block_mapping_key(emitter, event, true) + + case yaml_EMIT_BLOCK_MAPPING_KEY_STATE: + return yaml_emitter_emit_block_mapping_key(emitter, event, false) + + case yaml_EMIT_BLOCK_MAPPING_SIMPLE_VALUE_STATE: + return yaml_emitter_emit_block_mapping_value(emitter, event, true) + + case yaml_EMIT_BLOCK_MAPPING_VALUE_STATE: + return yaml_emitter_emit_block_mapping_value(emitter, event, false) + + case yaml_EMIT_END_STATE: + return yaml_emitter_set_emitter_error(emitter, "expected nothing after STREAM-END") + } + panic("invalid emitter state") +} + +// Expect STREAM-START. +func yaml_emitter_emit_stream_start(emitter *yaml_emitter_t, event *yaml_event_t) bool { + if event.typ != yaml_STREAM_START_EVENT { + return yaml_emitter_set_emitter_error(emitter, "expected STREAM-START") + } + if emitter.encoding == yaml_ANY_ENCODING { + emitter.encoding = event.encoding + if emitter.encoding == yaml_ANY_ENCODING { + emitter.encoding = yaml_UTF8_ENCODING + } + } + if emitter.best_indent < 2 || emitter.best_indent > 9 { + emitter.best_indent = 2 + } + if emitter.best_width >= 0 && emitter.best_width <= emitter.best_indent*2 { + emitter.best_width = 80 + } + if emitter.best_width < 0 { + emitter.best_width = 1<<31 - 1 + } + if emitter.line_break == yaml_ANY_BREAK { + emitter.line_break = yaml_LN_BREAK + } + + emitter.indent = -1 + emitter.line = 0 + emitter.column = 0 + emitter.whitespace = true + emitter.indention = true + + if emitter.encoding != yaml_UTF8_ENCODING { + if !yaml_emitter_write_bom(emitter) { + return false + } + } + emitter.state = yaml_EMIT_FIRST_DOCUMENT_START_STATE + return true +} + +// Expect DOCUMENT-START or STREAM-END. +func yaml_emitter_emit_document_start(emitter *yaml_emitter_t, event *yaml_event_t, first bool) bool { + + if event.typ == yaml_DOCUMENT_START_EVENT { + + if event.version_directive != nil { + if !yaml_emitter_analyze_version_directive(emitter, event.version_directive) { + return false + } + } + + for i := 0; i < len(event.tag_directives); i++ { + tag_directive := &event.tag_directives[i] + if !yaml_emitter_analyze_tag_directive(emitter, tag_directive) { + return false + } + if !yaml_emitter_append_tag_directive(emitter, tag_directive, false) { + return false + } + } + + for i := 0; i < len(default_tag_directives); i++ { + tag_directive := &default_tag_directives[i] + if !yaml_emitter_append_tag_directive(emitter, tag_directive, true) { + return false + } + } + + implicit := event.implicit + if !first || emitter.canonical { + implicit = false + } + + if emitter.open_ended && (event.version_directive != nil || len(event.tag_directives) > 0) { + if !yaml_emitter_write_indicator(emitter, []byte("..."), true, false, false) { + return false + } + if !yaml_emitter_write_indent(emitter) { + return false + } + } + + if event.version_directive != nil { + implicit = false + if !yaml_emitter_write_indicator(emitter, []byte("%YAML"), true, false, false) { + return false + } + if !yaml_emitter_write_indicator(emitter, []byte("1.1"), true, false, false) { + return false + } + if !yaml_emitter_write_indent(emitter) { + return false + } + } + + if len(event.tag_directives) > 0 { + implicit = false + for i := 0; i < len(event.tag_directives); i++ { + tag_directive := &event.tag_directives[i] + if !yaml_emitter_write_indicator(emitter, []byte("%TAG"), true, false, false) { + return false + } + if !yaml_emitter_write_tag_handle(emitter, tag_directive.handle) { + return false + } + if !yaml_emitter_write_tag_content(emitter, tag_directive.prefix, true) { + return false + } + if !yaml_emitter_write_indent(emitter) { + return false + } + } + } + + if yaml_emitter_check_empty_document(emitter) { + implicit = false + } + if !implicit { + if !yaml_emitter_write_indent(emitter) { + return false + } + if !yaml_emitter_write_indicator(emitter, []byte("---"), true, false, false) { + return false + } + if emitter.canonical { + if !yaml_emitter_write_indent(emitter) { + return false + } + } + } + + emitter.state = yaml_EMIT_DOCUMENT_CONTENT_STATE + return true + } + + if event.typ == yaml_STREAM_END_EVENT { + if emitter.open_ended { + if !yaml_emitter_write_indicator(emitter, []byte("..."), true, false, false) { + return false + } + if !yaml_emitter_write_indent(emitter) { + return false + } + } + if !yaml_emitter_flush(emitter) { + return false + } + emitter.state = yaml_EMIT_END_STATE + return true + } + + return yaml_emitter_set_emitter_error(emitter, "expected DOCUMENT-START or STREAM-END") +} + +// Expect the root node. +func yaml_emitter_emit_document_content(emitter *yaml_emitter_t, event *yaml_event_t) bool { + emitter.states = append(emitter.states, yaml_EMIT_DOCUMENT_END_STATE) + return yaml_emitter_emit_node(emitter, event, true, false, false, false) +} + +// Expect DOCUMENT-END. +func yaml_emitter_emit_document_end(emitter *yaml_emitter_t, event *yaml_event_t) bool { + if event.typ != yaml_DOCUMENT_END_EVENT { + return yaml_emitter_set_emitter_error(emitter, "expected DOCUMENT-END") + } + if !yaml_emitter_write_indent(emitter) { + return false + } + if !event.implicit { + // [Go] Allocate the slice elsewhere. + if !yaml_emitter_write_indicator(emitter, []byte("..."), true, false, false) { + return false + } + if !yaml_emitter_write_indent(emitter) { + return false + } + } + if !yaml_emitter_flush(emitter) { + return false + } + emitter.state = yaml_EMIT_DOCUMENT_START_STATE + emitter.tag_directives = emitter.tag_directives[:0] + return true +} + +// Expect a flow item node. +func yaml_emitter_emit_flow_sequence_item(emitter *yaml_emitter_t, event *yaml_event_t, first bool) bool { + if first { + if !yaml_emitter_write_indicator(emitter, []byte{'['}, true, true, false) { + return false + } + if !yaml_emitter_increase_indent(emitter, true, false) { + return false + } + emitter.flow_level++ + } + + if event.typ == yaml_SEQUENCE_END_EVENT { + emitter.flow_level-- + emitter.indent = emitter.indents[len(emitter.indents)-1] + emitter.indents = emitter.indents[:len(emitter.indents)-1] + if emitter.canonical && !first { + if !yaml_emitter_write_indicator(emitter, []byte{','}, false, false, false) { + return false + } + if !yaml_emitter_write_indent(emitter) { + return false + } + } + if !yaml_emitter_write_indicator(emitter, []byte{']'}, false, false, false) { + return false + } + emitter.state = emitter.states[len(emitter.states)-1] + emitter.states = emitter.states[:len(emitter.states)-1] + + return true + } + + if !first { + if !yaml_emitter_write_indicator(emitter, []byte{','}, false, false, false) { + return false + } + } + + if emitter.canonical || emitter.column > emitter.best_width { + if !yaml_emitter_write_indent(emitter) { + return false + } + } + emitter.states = append(emitter.states, yaml_EMIT_FLOW_SEQUENCE_ITEM_STATE) + return yaml_emitter_emit_node(emitter, event, false, true, false, false) +} + +// Expect a flow key node. +func yaml_emitter_emit_flow_mapping_key(emitter *yaml_emitter_t, event *yaml_event_t, first bool) bool { + if first { + if !yaml_emitter_write_indicator(emitter, []byte{'{'}, true, true, false) { + return false + } + if !yaml_emitter_increase_indent(emitter, true, false) { + return false + } + emitter.flow_level++ + } + + if event.typ == yaml_MAPPING_END_EVENT { + emitter.flow_level-- + emitter.indent = emitter.indents[len(emitter.indents)-1] + emitter.indents = emitter.indents[:len(emitter.indents)-1] + if emitter.canonical && !first { + if !yaml_emitter_write_indicator(emitter, []byte{','}, false, false, false) { + return false + } + if !yaml_emitter_write_indent(emitter) { + return false + } + } + if !yaml_emitter_write_indicator(emitter, []byte{'}'}, false, false, false) { + return false + } + emitter.state = emitter.states[len(emitter.states)-1] + emitter.states = emitter.states[:len(emitter.states)-1] + return true + } + + if !first { + if !yaml_emitter_write_indicator(emitter, []byte{','}, false, false, false) { + return false + } + } + if emitter.canonical || emitter.column > emitter.best_width { + if !yaml_emitter_write_indent(emitter) { + return false + } + } + + if !emitter.canonical && yaml_emitter_check_simple_key(emitter) { + emitter.states = append(emitter.states, yaml_EMIT_FLOW_MAPPING_SIMPLE_VALUE_STATE) + return yaml_emitter_emit_node(emitter, event, false, false, true, true) + } + if !yaml_emitter_write_indicator(emitter, []byte{'?'}, true, false, false) { + return false + } + emitter.states = append(emitter.states, yaml_EMIT_FLOW_MAPPING_VALUE_STATE) + return yaml_emitter_emit_node(emitter, event, false, false, true, false) +} + +// Expect a flow value node. +func yaml_emitter_emit_flow_mapping_value(emitter *yaml_emitter_t, event *yaml_event_t, simple bool) bool { + if simple { + if !yaml_emitter_write_indicator(emitter, []byte{':'}, false, false, false) { + return false + } + } else { + if emitter.canonical || emitter.column > emitter.best_width { + if !yaml_emitter_write_indent(emitter) { + return false + } + } + if !yaml_emitter_write_indicator(emitter, []byte{':'}, true, false, false) { + return false + } + } + emitter.states = append(emitter.states, yaml_EMIT_FLOW_MAPPING_KEY_STATE) + return yaml_emitter_emit_node(emitter, event, false, false, true, false) +} + +// Expect a block item node. +func yaml_emitter_emit_block_sequence_item(emitter *yaml_emitter_t, event *yaml_event_t, first bool) bool { + if first { + if !yaml_emitter_increase_indent(emitter, false, emitter.mapping_context && !emitter.indention) { + return false + } + } + if event.typ == yaml_SEQUENCE_END_EVENT { + emitter.indent = emitter.indents[len(emitter.indents)-1] + emitter.indents = emitter.indents[:len(emitter.indents)-1] + emitter.state = emitter.states[len(emitter.states)-1] + emitter.states = emitter.states[:len(emitter.states)-1] + return true + } + if !yaml_emitter_write_indent(emitter) { + return false + } + if !yaml_emitter_write_indicator(emitter, []byte{'-'}, true, false, true) { + return false + } + emitter.states = append(emitter.states, yaml_EMIT_BLOCK_SEQUENCE_ITEM_STATE) + return yaml_emitter_emit_node(emitter, event, false, true, false, false) +} + +// Expect a block key node. +func yaml_emitter_emit_block_mapping_key(emitter *yaml_emitter_t, event *yaml_event_t, first bool) bool { + if first { + if !yaml_emitter_increase_indent(emitter, false, false) { + return false + } + } + if event.typ == yaml_MAPPING_END_EVENT { + emitter.indent = emitter.indents[len(emitter.indents)-1] + emitter.indents = emitter.indents[:len(emitter.indents)-1] + emitter.state = emitter.states[len(emitter.states)-1] + emitter.states = emitter.states[:len(emitter.states)-1] + return true + } + if !yaml_emitter_write_indent(emitter) { + return false + } + if yaml_emitter_check_simple_key(emitter) { + emitter.states = append(emitter.states, yaml_EMIT_BLOCK_MAPPING_SIMPLE_VALUE_STATE) + return yaml_emitter_emit_node(emitter, event, false, false, true, true) + } + if !yaml_emitter_write_indicator(emitter, []byte{'?'}, true, false, true) { + return false + } + emitter.states = append(emitter.states, yaml_EMIT_BLOCK_MAPPING_VALUE_STATE) + return yaml_emitter_emit_node(emitter, event, false, false, true, false) +} + +// Expect a block value node. +func yaml_emitter_emit_block_mapping_value(emitter *yaml_emitter_t, event *yaml_event_t, simple bool) bool { + if simple { + if !yaml_emitter_write_indicator(emitter, []byte{':'}, false, false, false) { + return false + } + } else { + if !yaml_emitter_write_indent(emitter) { + return false + } + if !yaml_emitter_write_indicator(emitter, []byte{':'}, true, false, true) { + return false + } + } + emitter.states = append(emitter.states, yaml_EMIT_BLOCK_MAPPING_KEY_STATE) + return yaml_emitter_emit_node(emitter, event, false, false, true, false) +} + +// Expect a node. +func yaml_emitter_emit_node(emitter *yaml_emitter_t, event *yaml_event_t, + root bool, sequence bool, mapping bool, simple_key bool) bool { + + emitter.root_context = root + emitter.sequence_context = sequence + emitter.mapping_context = mapping + emitter.simple_key_context = simple_key + + switch event.typ { + case yaml_ALIAS_EVENT: + return yaml_emitter_emit_alias(emitter, event) + case yaml_SCALAR_EVENT: + return yaml_emitter_emit_scalar(emitter, event) + case yaml_SEQUENCE_START_EVENT: + return yaml_emitter_emit_sequence_start(emitter, event) + case yaml_MAPPING_START_EVENT: + return yaml_emitter_emit_mapping_start(emitter, event) + default: + return yaml_emitter_set_emitter_error(emitter, + fmt.Sprintf("expected SCALAR, SEQUENCE-START, MAPPING-START, or ALIAS, but got %v", event.typ)) + } +} + +// Expect ALIAS. +func yaml_emitter_emit_alias(emitter *yaml_emitter_t, event *yaml_event_t) bool { + if !yaml_emitter_process_anchor(emitter) { + return false + } + emitter.state = emitter.states[len(emitter.states)-1] + emitter.states = emitter.states[:len(emitter.states)-1] + return true +} + +// Expect SCALAR. +func yaml_emitter_emit_scalar(emitter *yaml_emitter_t, event *yaml_event_t) bool { + if !yaml_emitter_select_scalar_style(emitter, event) { + return false + } + if !yaml_emitter_process_anchor(emitter) { + return false + } + if !yaml_emitter_process_tag(emitter) { + return false + } + if !yaml_emitter_increase_indent(emitter, true, false) { + return false + } + if !yaml_emitter_process_scalar(emitter) { + return false + } + emitter.indent = emitter.indents[len(emitter.indents)-1] + emitter.indents = emitter.indents[:len(emitter.indents)-1] + emitter.state = emitter.states[len(emitter.states)-1] + emitter.states = emitter.states[:len(emitter.states)-1] + return true +} + +// Expect SEQUENCE-START. +func yaml_emitter_emit_sequence_start(emitter *yaml_emitter_t, event *yaml_event_t) bool { + if !yaml_emitter_process_anchor(emitter) { + return false + } + if !yaml_emitter_process_tag(emitter) { + return false + } + if emitter.flow_level > 0 || emitter.canonical || event.sequence_style() == yaml_FLOW_SEQUENCE_STYLE || + yaml_emitter_check_empty_sequence(emitter) { + emitter.state = yaml_EMIT_FLOW_SEQUENCE_FIRST_ITEM_STATE + } else { + emitter.state = yaml_EMIT_BLOCK_SEQUENCE_FIRST_ITEM_STATE + } + return true +} + +// Expect MAPPING-START. +func yaml_emitter_emit_mapping_start(emitter *yaml_emitter_t, event *yaml_event_t) bool { + if !yaml_emitter_process_anchor(emitter) { + return false + } + if !yaml_emitter_process_tag(emitter) { + return false + } + if emitter.flow_level > 0 || emitter.canonical || event.mapping_style() == yaml_FLOW_MAPPING_STYLE || + yaml_emitter_check_empty_mapping(emitter) { + emitter.state = yaml_EMIT_FLOW_MAPPING_FIRST_KEY_STATE + } else { + emitter.state = yaml_EMIT_BLOCK_MAPPING_FIRST_KEY_STATE + } + return true +} + +// Check if the document content is an empty scalar. +func yaml_emitter_check_empty_document(emitter *yaml_emitter_t) bool { + return false // [Go] Huh? +} + +// Check if the next events represent an empty sequence. +func yaml_emitter_check_empty_sequence(emitter *yaml_emitter_t) bool { + if len(emitter.events)-emitter.events_head < 2 { + return false + } + return emitter.events[emitter.events_head].typ == yaml_SEQUENCE_START_EVENT && + emitter.events[emitter.events_head+1].typ == yaml_SEQUENCE_END_EVENT +} + +// Check if the next events represent an empty mapping. +func yaml_emitter_check_empty_mapping(emitter *yaml_emitter_t) bool { + if len(emitter.events)-emitter.events_head < 2 { + return false + } + return emitter.events[emitter.events_head].typ == yaml_MAPPING_START_EVENT && + emitter.events[emitter.events_head+1].typ == yaml_MAPPING_END_EVENT +} + +// Check if the next node can be expressed as a simple key. +func yaml_emitter_check_simple_key(emitter *yaml_emitter_t) bool { + length := 0 + switch emitter.events[emitter.events_head].typ { + case yaml_ALIAS_EVENT: + length += len(emitter.anchor_data.anchor) + case yaml_SCALAR_EVENT: + if emitter.scalar_data.multiline { + return false + } + length += len(emitter.anchor_data.anchor) + + len(emitter.tag_data.handle) + + len(emitter.tag_data.suffix) + + len(emitter.scalar_data.value) + case yaml_SEQUENCE_START_EVENT: + if !yaml_emitter_check_empty_sequence(emitter) { + return false + } + length += len(emitter.anchor_data.anchor) + + len(emitter.tag_data.handle) + + len(emitter.tag_data.suffix) + case yaml_MAPPING_START_EVENT: + if !yaml_emitter_check_empty_mapping(emitter) { + return false + } + length += len(emitter.anchor_data.anchor) + + len(emitter.tag_data.handle) + + len(emitter.tag_data.suffix) + default: + return false + } + return length <= 128 +} + +// Determine an acceptable scalar style. +func yaml_emitter_select_scalar_style(emitter *yaml_emitter_t, event *yaml_event_t) bool { + + no_tag := len(emitter.tag_data.handle) == 0 && len(emitter.tag_data.suffix) == 0 + if no_tag && !event.implicit && !event.quoted_implicit { + return yaml_emitter_set_emitter_error(emitter, "neither tag nor implicit flags are specified") + } + + style := event.scalar_style() + if style == yaml_ANY_SCALAR_STYLE { + style = yaml_PLAIN_SCALAR_STYLE + } + if emitter.canonical { + style = yaml_DOUBLE_QUOTED_SCALAR_STYLE + } + if emitter.simple_key_context && emitter.scalar_data.multiline { + style = yaml_DOUBLE_QUOTED_SCALAR_STYLE + } + + if style == yaml_PLAIN_SCALAR_STYLE { + if emitter.flow_level > 0 && !emitter.scalar_data.flow_plain_allowed || + emitter.flow_level == 0 && !emitter.scalar_data.block_plain_allowed { + style = yaml_SINGLE_QUOTED_SCALAR_STYLE + } + if len(emitter.scalar_data.value) == 0 && (emitter.flow_level > 0 || emitter.simple_key_context) { + style = yaml_SINGLE_QUOTED_SCALAR_STYLE + } + if no_tag && !event.implicit { + style = yaml_SINGLE_QUOTED_SCALAR_STYLE + } + } + if style == yaml_SINGLE_QUOTED_SCALAR_STYLE { + if !emitter.scalar_data.single_quoted_allowed { + style = yaml_DOUBLE_QUOTED_SCALAR_STYLE + } + } + if style == yaml_LITERAL_SCALAR_STYLE || style == yaml_FOLDED_SCALAR_STYLE { + if !emitter.scalar_data.block_allowed || emitter.flow_level > 0 || emitter.simple_key_context { + style = yaml_DOUBLE_QUOTED_SCALAR_STYLE + } + } + + if no_tag && !event.quoted_implicit && style != yaml_PLAIN_SCALAR_STYLE { + emitter.tag_data.handle = []byte{'!'} + } + emitter.scalar_data.style = style + return true +} + +// Write an anchor. +func yaml_emitter_process_anchor(emitter *yaml_emitter_t) bool { + if emitter.anchor_data.anchor == nil { + return true + } + c := []byte{'&'} + if emitter.anchor_data.alias { + c[0] = '*' + } + if !yaml_emitter_write_indicator(emitter, c, true, false, false) { + return false + } + return yaml_emitter_write_anchor(emitter, emitter.anchor_data.anchor) +} + +// Write a tag. +func yaml_emitter_process_tag(emitter *yaml_emitter_t) bool { + if len(emitter.tag_data.handle) == 0 && len(emitter.tag_data.suffix) == 0 { + return true + } + if len(emitter.tag_data.handle) > 0 { + if !yaml_emitter_write_tag_handle(emitter, emitter.tag_data.handle) { + return false + } + if len(emitter.tag_data.suffix) > 0 { + if !yaml_emitter_write_tag_content(emitter, emitter.tag_data.suffix, false) { + return false + } + } + } else { + // [Go] Allocate these slices elsewhere. + if !yaml_emitter_write_indicator(emitter, []byte("!<"), true, false, false) { + return false + } + if !yaml_emitter_write_tag_content(emitter, emitter.tag_data.suffix, false) { + return false + } + if !yaml_emitter_write_indicator(emitter, []byte{'>'}, false, false, false) { + return false + } + } + return true +} + +// Write a scalar. +func yaml_emitter_process_scalar(emitter *yaml_emitter_t) bool { + switch emitter.scalar_data.style { + case yaml_PLAIN_SCALAR_STYLE: + return yaml_emitter_write_plain_scalar(emitter, emitter.scalar_data.value, !emitter.simple_key_context) + + case yaml_SINGLE_QUOTED_SCALAR_STYLE: + return yaml_emitter_write_single_quoted_scalar(emitter, emitter.scalar_data.value, !emitter.simple_key_context) + + case yaml_DOUBLE_QUOTED_SCALAR_STYLE: + return yaml_emitter_write_double_quoted_scalar(emitter, emitter.scalar_data.value, !emitter.simple_key_context) + + case yaml_LITERAL_SCALAR_STYLE: + return yaml_emitter_write_literal_scalar(emitter, emitter.scalar_data.value) + + case yaml_FOLDED_SCALAR_STYLE: + return yaml_emitter_write_folded_scalar(emitter, emitter.scalar_data.value) + } + panic("unknown scalar style") +} + +// Check if a %YAML directive is valid. +func yaml_emitter_analyze_version_directive(emitter *yaml_emitter_t, version_directive *yaml_version_directive_t) bool { + if version_directive.major != 1 || version_directive.minor != 1 { + return yaml_emitter_set_emitter_error(emitter, "incompatible %YAML directive") + } + return true +} + +// Check if a %TAG directive is valid. +func yaml_emitter_analyze_tag_directive(emitter *yaml_emitter_t, tag_directive *yaml_tag_directive_t) bool { + handle := tag_directive.handle + prefix := tag_directive.prefix + if len(handle) == 0 { + return yaml_emitter_set_emitter_error(emitter, "tag handle must not be empty") + } + if handle[0] != '!' { + return yaml_emitter_set_emitter_error(emitter, "tag handle must start with '!'") + } + if handle[len(handle)-1] != '!' { + return yaml_emitter_set_emitter_error(emitter, "tag handle must end with '!'") + } + for i := 1; i < len(handle)-1; i += width(handle[i]) { + if !is_alpha(handle, i) { + return yaml_emitter_set_emitter_error(emitter, "tag handle must contain alphanumerical characters only") + } + } + if len(prefix) == 0 { + return yaml_emitter_set_emitter_error(emitter, "tag prefix must not be empty") + } + return true +} + +// Check if an anchor is valid. +func yaml_emitter_analyze_anchor(emitter *yaml_emitter_t, anchor []byte, alias bool) bool { + if len(anchor) == 0 { + problem := "anchor value must not be empty" + if alias { + problem = "alias value must not be empty" + } + return yaml_emitter_set_emitter_error(emitter, problem) + } + for i := 0; i < len(anchor); i += width(anchor[i]) { + if !is_alpha(anchor, i) { + problem := "anchor value must contain alphanumerical characters only" + if alias { + problem = "alias value must contain alphanumerical characters only" + } + return yaml_emitter_set_emitter_error(emitter, problem) + } + } + emitter.anchor_data.anchor = anchor + emitter.anchor_data.alias = alias + return true +} + +// Check if a tag is valid. +func yaml_emitter_analyze_tag(emitter *yaml_emitter_t, tag []byte) bool { + if len(tag) == 0 { + return yaml_emitter_set_emitter_error(emitter, "tag value must not be empty") + } + for i := 0; i < len(emitter.tag_directives); i++ { + tag_directive := &emitter.tag_directives[i] + if bytes.HasPrefix(tag, tag_directive.prefix) { + emitter.tag_data.handle = tag_directive.handle + emitter.tag_data.suffix = tag[len(tag_directive.prefix):] + return true + } + } + emitter.tag_data.suffix = tag + return true +} + +// Check if a scalar is valid. +func yaml_emitter_analyze_scalar(emitter *yaml_emitter_t, value []byte) bool { + var ( + block_indicators = false + flow_indicators = false + line_breaks = false + special_characters = false + + leading_space = false + leading_break = false + trailing_space = false + trailing_break = false + break_space = false + space_break = false + + preceded_by_whitespace = false + followed_by_whitespace = false + previous_space = false + previous_break = false + ) + + emitter.scalar_data.value = value + + if len(value) == 0 { + emitter.scalar_data.multiline = false + emitter.scalar_data.flow_plain_allowed = false + emitter.scalar_data.block_plain_allowed = true + emitter.scalar_data.single_quoted_allowed = true + emitter.scalar_data.block_allowed = false + return true + } + + if len(value) >= 3 && ((value[0] == '-' && value[1] == '-' && value[2] == '-') || (value[0] == '.' && value[1] == '.' && value[2] == '.')) { + block_indicators = true + flow_indicators = true + } + + preceded_by_whitespace = true + for i, w := 0, 0; i < len(value); i += w { + w = width(value[i]) + followed_by_whitespace = i+w >= len(value) || is_blank(value, i+w) + + if i == 0 { + switch value[i] { + case '#', ',', '[', ']', '{', '}', '&', '*', '!', '|', '>', '\'', '"', '%', '@', '`': + flow_indicators = true + block_indicators = true + case '?', ':': + flow_indicators = true + if followed_by_whitespace { + block_indicators = true + } + case '-': + if followed_by_whitespace { + flow_indicators = true + block_indicators = true + } + } + } else { + switch value[i] { + case ',', '?', '[', ']', '{', '}': + flow_indicators = true + case ':': + flow_indicators = true + if followed_by_whitespace { + block_indicators = true + } + case '#': + if preceded_by_whitespace { + flow_indicators = true + block_indicators = true + } + } + } + + if !is_printable(value, i) || !is_ascii(value, i) && !emitter.unicode { + special_characters = true + } + if is_space(value, i) { + if i == 0 { + leading_space = true + } + if i+width(value[i]) == len(value) { + trailing_space = true + } + if previous_break { + break_space = true + } + previous_space = true + previous_break = false + } else if is_break(value, i) { + line_breaks = true + if i == 0 { + leading_break = true + } + if i+width(value[i]) == len(value) { + trailing_break = true + } + if previous_space { + space_break = true + } + previous_space = false + previous_break = true + } else { + previous_space = false + previous_break = false + } + + // [Go]: Why 'z'? Couldn't be the end of the string as that's the loop condition. + preceded_by_whitespace = is_blankz(value, i) + } + + emitter.scalar_data.multiline = line_breaks + emitter.scalar_data.flow_plain_allowed = true + emitter.scalar_data.block_plain_allowed = true + emitter.scalar_data.single_quoted_allowed = true + emitter.scalar_data.block_allowed = true + + if leading_space || leading_break || trailing_space || trailing_break { + emitter.scalar_data.flow_plain_allowed = false + emitter.scalar_data.block_plain_allowed = false + } + if trailing_space { + emitter.scalar_data.block_allowed = false + } + if break_space { + emitter.scalar_data.flow_plain_allowed = false + emitter.scalar_data.block_plain_allowed = false + emitter.scalar_data.single_quoted_allowed = false + } + if space_break || special_characters { + emitter.scalar_data.flow_plain_allowed = false + emitter.scalar_data.block_plain_allowed = false + emitter.scalar_data.single_quoted_allowed = false + emitter.scalar_data.block_allowed = false + } + if line_breaks { + emitter.scalar_data.flow_plain_allowed = false + emitter.scalar_data.block_plain_allowed = false + } + if flow_indicators { + emitter.scalar_data.flow_plain_allowed = false + } + if block_indicators { + emitter.scalar_data.block_plain_allowed = false + } + return true +} + +// Check if the event data is valid. +func yaml_emitter_analyze_event(emitter *yaml_emitter_t, event *yaml_event_t) bool { + + emitter.anchor_data.anchor = nil + emitter.tag_data.handle = nil + emitter.tag_data.suffix = nil + emitter.scalar_data.value = nil + + switch event.typ { + case yaml_ALIAS_EVENT: + if !yaml_emitter_analyze_anchor(emitter, event.anchor, true) { + return false + } + + case yaml_SCALAR_EVENT: + if len(event.anchor) > 0 { + if !yaml_emitter_analyze_anchor(emitter, event.anchor, false) { + return false + } + } + if len(event.tag) > 0 && (emitter.canonical || (!event.implicit && !event.quoted_implicit)) { + if !yaml_emitter_analyze_tag(emitter, event.tag) { + return false + } + } + if !yaml_emitter_analyze_scalar(emitter, event.value) { + return false + } + + case yaml_SEQUENCE_START_EVENT: + if len(event.anchor) > 0 { + if !yaml_emitter_analyze_anchor(emitter, event.anchor, false) { + return false + } + } + if len(event.tag) > 0 && (emitter.canonical || !event.implicit) { + if !yaml_emitter_analyze_tag(emitter, event.tag) { + return false + } + } + + case yaml_MAPPING_START_EVENT: + if len(event.anchor) > 0 { + if !yaml_emitter_analyze_anchor(emitter, event.anchor, false) { + return false + } + } + if len(event.tag) > 0 && (emitter.canonical || !event.implicit) { + if !yaml_emitter_analyze_tag(emitter, event.tag) { + return false + } + } + } + return true +} + +// Write the BOM character. +func yaml_emitter_write_bom(emitter *yaml_emitter_t) bool { + if !flush(emitter) { + return false + } + pos := emitter.buffer_pos + emitter.buffer[pos+0] = '\xEF' + emitter.buffer[pos+1] = '\xBB' + emitter.buffer[pos+2] = '\xBF' + emitter.buffer_pos += 3 + return true +} + +func yaml_emitter_write_indent(emitter *yaml_emitter_t) bool { + indent := emitter.indent + if indent < 0 { + indent = 0 + } + if !emitter.indention || emitter.column > indent || (emitter.column == indent && !emitter.whitespace) { + if !put_break(emitter) { + return false + } + } + for emitter.column < indent { + if !put(emitter, ' ') { + return false + } + } + emitter.whitespace = true + emitter.indention = true + return true +} + +func yaml_emitter_write_indicator(emitter *yaml_emitter_t, indicator []byte, need_whitespace, is_whitespace, is_indention bool) bool { + if need_whitespace && !emitter.whitespace { + if !put(emitter, ' ') { + return false + } + } + if !write_all(emitter, indicator) { + return false + } + emitter.whitespace = is_whitespace + emitter.indention = (emitter.indention && is_indention) + emitter.open_ended = false + return true +} + +func yaml_emitter_write_anchor(emitter *yaml_emitter_t, value []byte) bool { + if !write_all(emitter, value) { + return false + } + emitter.whitespace = false + emitter.indention = false + return true +} + +func yaml_emitter_write_tag_handle(emitter *yaml_emitter_t, value []byte) bool { + if !emitter.whitespace { + if !put(emitter, ' ') { + return false + } + } + if !write_all(emitter, value) { + return false + } + emitter.whitespace = false + emitter.indention = false + return true +} + +func yaml_emitter_write_tag_content(emitter *yaml_emitter_t, value []byte, need_whitespace bool) bool { + if need_whitespace && !emitter.whitespace { + if !put(emitter, ' ') { + return false + } + } + for i := 0; i < len(value); { + var must_write bool + switch value[i] { + case ';', '/', '?', ':', '@', '&', '=', '+', '$', ',', '_', '.', '~', '*', '\'', '(', ')', '[', ']': + must_write = true + default: + must_write = is_alpha(value, i) + } + if must_write { + if !write(emitter, value, &i) { + return false + } + } else { + w := width(value[i]) + for k := 0; k < w; k++ { + octet := value[i] + i++ + if !put(emitter, '%') { + return false + } + + c := octet >> 4 + if c < 10 { + c += '0' + } else { + c += 'A' - 10 + } + if !put(emitter, c) { + return false + } + + c = octet & 0x0f + if c < 10 { + c += '0' + } else { + c += 'A' - 10 + } + if !put(emitter, c) { + return false + } + } + } + } + emitter.whitespace = false + emitter.indention = false + return true +} + +func yaml_emitter_write_plain_scalar(emitter *yaml_emitter_t, value []byte, allow_breaks bool) bool { + if !emitter.whitespace { + if !put(emitter, ' ') { + return false + } + } + + spaces := false + breaks := false + for i := 0; i < len(value); { + if is_space(value, i) { + if allow_breaks && !spaces && emitter.column > emitter.best_width && !is_space(value, i+1) { + if !yaml_emitter_write_indent(emitter) { + return false + } + i += width(value[i]) + } else { + if !write(emitter, value, &i) { + return false + } + } + spaces = true + } else if is_break(value, i) { + if !breaks && value[i] == '\n' { + if !put_break(emitter) { + return false + } + } + if !write_break(emitter, value, &i) { + return false + } + emitter.indention = true + breaks = true + } else { + if breaks { + if !yaml_emitter_write_indent(emitter) { + return false + } + } + if !write(emitter, value, &i) { + return false + } + emitter.indention = false + spaces = false + breaks = false + } + } + + emitter.whitespace = false + emitter.indention = false + if emitter.root_context { + emitter.open_ended = true + } + + return true +} + +func yaml_emitter_write_single_quoted_scalar(emitter *yaml_emitter_t, value []byte, allow_breaks bool) bool { + + if !yaml_emitter_write_indicator(emitter, []byte{'\''}, true, false, false) { + return false + } + + spaces := false + breaks := false + for i := 0; i < len(value); { + if is_space(value, i) { + if allow_breaks && !spaces && emitter.column > emitter.best_width && i > 0 && i < len(value)-1 && !is_space(value, i+1) { + if !yaml_emitter_write_indent(emitter) { + return false + } + i += width(value[i]) + } else { + if !write(emitter, value, &i) { + return false + } + } + spaces = true + } else if is_break(value, i) { + if !breaks && value[i] == '\n' { + if !put_break(emitter) { + return false + } + } + if !write_break(emitter, value, &i) { + return false + } + emitter.indention = true + breaks = true + } else { + if breaks { + if !yaml_emitter_write_indent(emitter) { + return false + } + } + if value[i] == '\'' { + if !put(emitter, '\'') { + return false + } + } + if !write(emitter, value, &i) { + return false + } + emitter.indention = false + spaces = false + breaks = false + } + } + if !yaml_emitter_write_indicator(emitter, []byte{'\''}, false, false, false) { + return false + } + emitter.whitespace = false + emitter.indention = false + return true +} + +func yaml_emitter_write_double_quoted_scalar(emitter *yaml_emitter_t, value []byte, allow_breaks bool) bool { + spaces := false + if !yaml_emitter_write_indicator(emitter, []byte{'"'}, true, false, false) { + return false + } + + for i := 0; i < len(value); { + if !is_printable(value, i) || (!emitter.unicode && !is_ascii(value, i)) || + is_bom(value, i) || is_break(value, i) || + value[i] == '"' || value[i] == '\\' { + + octet := value[i] + + var w int + var v rune + switch { + case octet&0x80 == 0x00: + w, v = 1, rune(octet&0x7F) + case octet&0xE0 == 0xC0: + w, v = 2, rune(octet&0x1F) + case octet&0xF0 == 0xE0: + w, v = 3, rune(octet&0x0F) + case octet&0xF8 == 0xF0: + w, v = 4, rune(octet&0x07) + } + for k := 1; k < w; k++ { + octet = value[i+k] + v = (v << 6) + (rune(octet) & 0x3F) + } + i += w + + if !put(emitter, '\\') { + return false + } + + var ok bool + switch v { + case 0x00: + ok = put(emitter, '0') + case 0x07: + ok = put(emitter, 'a') + case 0x08: + ok = put(emitter, 'b') + case 0x09: + ok = put(emitter, 't') + case 0x0A: + ok = put(emitter, 'n') + case 0x0b: + ok = put(emitter, 'v') + case 0x0c: + ok = put(emitter, 'f') + case 0x0d: + ok = put(emitter, 'r') + case 0x1b: + ok = put(emitter, 'e') + case 0x22: + ok = put(emitter, '"') + case 0x5c: + ok = put(emitter, '\\') + case 0x85: + ok = put(emitter, 'N') + case 0xA0: + ok = put(emitter, '_') + case 0x2028: + ok = put(emitter, 'L') + case 0x2029: + ok = put(emitter, 'P') + default: + if v <= 0xFF { + ok = put(emitter, 'x') + w = 2 + } else if v <= 0xFFFF { + ok = put(emitter, 'u') + w = 4 + } else { + ok = put(emitter, 'U') + w = 8 + } + for k := (w - 1) * 4; ok && k >= 0; k -= 4 { + digit := byte((v >> uint(k)) & 0x0F) + if digit < 10 { + ok = put(emitter, digit+'0') + } else { + ok = put(emitter, digit+'A'-10) + } + } + } + if !ok { + return false + } + spaces = false + } else if is_space(value, i) { + if allow_breaks && !spaces && emitter.column > emitter.best_width && i > 0 && i < len(value)-1 { + if !yaml_emitter_write_indent(emitter) { + return false + } + if is_space(value, i+1) { + if !put(emitter, '\\') { + return false + } + } + i += width(value[i]) + } else if !write(emitter, value, &i) { + return false + } + spaces = true + } else { + if !write(emitter, value, &i) { + return false + } + spaces = false + } + } + if !yaml_emitter_write_indicator(emitter, []byte{'"'}, false, false, false) { + return false + } + emitter.whitespace = false + emitter.indention = false + return true +} + +func yaml_emitter_write_block_scalar_hints(emitter *yaml_emitter_t, value []byte) bool { + if is_space(value, 0) || is_break(value, 0) { + indent_hint := []byte{'0' + byte(emitter.best_indent)} + if !yaml_emitter_write_indicator(emitter, indent_hint, false, false, false) { + return false + } + } + + emitter.open_ended = false + + var chomp_hint [1]byte + if len(value) == 0 { + chomp_hint[0] = '-' + } else { + i := len(value) - 1 + for value[i]&0xC0 == 0x80 { + i-- + } + if !is_break(value, i) { + chomp_hint[0] = '-' + } else if i == 0 { + chomp_hint[0] = '+' + emitter.open_ended = true + } else { + i-- + for value[i]&0xC0 == 0x80 { + i-- + } + if is_break(value, i) { + chomp_hint[0] = '+' + emitter.open_ended = true + } + } + } + if chomp_hint[0] != 0 { + if !yaml_emitter_write_indicator(emitter, chomp_hint[:], false, false, false) { + return false + } + } + return true +} + +func yaml_emitter_write_literal_scalar(emitter *yaml_emitter_t, value []byte) bool { + if !yaml_emitter_write_indicator(emitter, []byte{'|'}, true, false, false) { + return false + } + if !yaml_emitter_write_block_scalar_hints(emitter, value) { + return false + } + if !put_break(emitter) { + return false + } + emitter.indention = true + emitter.whitespace = true + breaks := true + for i := 0; i < len(value); { + if is_break(value, i) { + if !write_break(emitter, value, &i) { + return false + } + emitter.indention = true + breaks = true + } else { + if breaks { + if !yaml_emitter_write_indent(emitter) { + return false + } + } + if !write(emitter, value, &i) { + return false + } + emitter.indention = false + breaks = false + } + } + + return true +} + +func yaml_emitter_write_folded_scalar(emitter *yaml_emitter_t, value []byte) bool { + if !yaml_emitter_write_indicator(emitter, []byte{'>'}, true, false, false) { + return false + } + if !yaml_emitter_write_block_scalar_hints(emitter, value) { + return false + } + + if !put_break(emitter) { + return false + } + emitter.indention = true + emitter.whitespace = true + + breaks := true + leading_spaces := true + for i := 0; i < len(value); { + if is_break(value, i) { + if !breaks && !leading_spaces && value[i] == '\n' { + k := 0 + for is_break(value, k) { + k += width(value[k]) + } + if !is_blankz(value, k) { + if !put_break(emitter) { + return false + } + } + } + if !write_break(emitter, value, &i) { + return false + } + emitter.indention = true + breaks = true + } else { + if breaks { + if !yaml_emitter_write_indent(emitter) { + return false + } + leading_spaces = is_blank(value, i) + } + if !breaks && is_space(value, i) && !is_space(value, i+1) && emitter.column > emitter.best_width { + if !yaml_emitter_write_indent(emitter) { + return false + } + i += width(value[i]) + } else { + if !write(emitter, value, &i) { + return false + } + } + emitter.indention = false + breaks = false + } + } + return true +} diff --git a/tools/voluspa/vendor/gopkg.in/yaml.v2/encode.go b/tools/voluspa/vendor/gopkg.in/yaml.v2/encode.go new file mode 100644 index 00000000000..0ee738e11b6 --- /dev/null +++ b/tools/voluspa/vendor/gopkg.in/yaml.v2/encode.go @@ -0,0 +1,390 @@ +package yaml + +import ( + "encoding" + "fmt" + "io" + "reflect" + "regexp" + "sort" + "strconv" + "strings" + "time" + "unicode/utf8" +) + +// jsonNumber is the interface of the encoding/json.Number datatype. +// Repeating the interface here avoids a dependency on encoding/json, and also +// supports other libraries like jsoniter, which use a similar datatype with +// the same interface. Detecting this interface is useful when dealing with +// structures containing json.Number, which is a string under the hood. The +// encoder should prefer the use of Int64(), Float64() and string(), in that +// order, when encoding this type. +type jsonNumber interface { + Float64() (float64, error) + Int64() (int64, error) + String() string +} + +type encoder struct { + emitter yaml_emitter_t + event yaml_event_t + out []byte + flow bool + // doneInit holds whether the initial stream_start_event has been + // emitted. + doneInit bool +} + +func newEncoder() *encoder { + e := &encoder{} + yaml_emitter_initialize(&e.emitter) + yaml_emitter_set_output_string(&e.emitter, &e.out) + yaml_emitter_set_unicode(&e.emitter, true) + return e +} + +func newEncoderWithWriter(w io.Writer) *encoder { + e := &encoder{} + yaml_emitter_initialize(&e.emitter) + yaml_emitter_set_output_writer(&e.emitter, w) + yaml_emitter_set_unicode(&e.emitter, true) + return e +} + +func (e *encoder) init() { + if e.doneInit { + return + } + yaml_stream_start_event_initialize(&e.event, yaml_UTF8_ENCODING) + e.emit() + e.doneInit = true +} + +func (e *encoder) finish() { + e.emitter.open_ended = false + yaml_stream_end_event_initialize(&e.event) + e.emit() +} + +func (e *encoder) destroy() { + yaml_emitter_delete(&e.emitter) +} + +func (e *encoder) emit() { + // This will internally delete the e.event value. + e.must(yaml_emitter_emit(&e.emitter, &e.event)) +} + +func (e *encoder) must(ok bool) { + if !ok { + msg := e.emitter.problem + if msg == "" { + msg = "unknown problem generating YAML content" + } + failf("%s", msg) + } +} + +func (e *encoder) marshalDoc(tag string, in reflect.Value) { + e.init() + yaml_document_start_event_initialize(&e.event, nil, nil, true) + e.emit() + e.marshal(tag, in) + yaml_document_end_event_initialize(&e.event, true) + e.emit() +} + +func (e *encoder) marshal(tag string, in reflect.Value) { + if !in.IsValid() || in.Kind() == reflect.Ptr && in.IsNil() { + e.nilv() + return + } + iface := in.Interface() + switch m := iface.(type) { + case jsonNumber: + integer, err := m.Int64() + if err == nil { + // In this case the json.Number is a valid int64 + in = reflect.ValueOf(integer) + break + } + float, err := m.Float64() + if err == nil { + // In this case the json.Number is a valid float64 + in = reflect.ValueOf(float) + break + } + // fallback case - no number could be obtained + in = reflect.ValueOf(m.String()) + case time.Time, *time.Time: + // Although time.Time implements TextMarshaler, + // we don't want to treat it as a string for YAML + // purposes because YAML has special support for + // timestamps. + case Marshaler: + v, err := m.MarshalYAML() + if err != nil { + fail(err) + } + if v == nil { + e.nilv() + return + } + in = reflect.ValueOf(v) + case encoding.TextMarshaler: + text, err := m.MarshalText() + if err != nil { + fail(err) + } + in = reflect.ValueOf(string(text)) + case nil: + e.nilv() + return + } + switch in.Kind() { + case reflect.Interface: + e.marshal(tag, in.Elem()) + case reflect.Map: + e.mapv(tag, in) + case reflect.Ptr: + if in.Type() == ptrTimeType { + e.timev(tag, in.Elem()) + } else { + e.marshal(tag, in.Elem()) + } + case reflect.Struct: + if in.Type() == timeType { + e.timev(tag, in) + } else { + e.structv(tag, in) + } + case reflect.Slice, reflect.Array: + if in.Type().Elem() == mapItemType { + e.itemsv(tag, in) + } else { + e.slicev(tag, in) + } + case reflect.String: + e.stringv(tag, in) + case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64: + if in.Type() == durationType { + e.stringv(tag, reflect.ValueOf(iface.(time.Duration).String())) + } else { + e.intv(tag, in) + } + case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64, reflect.Uintptr: + e.uintv(tag, in) + case reflect.Float32, reflect.Float64: + e.floatv(tag, in) + case reflect.Bool: + e.boolv(tag, in) + default: + panic("cannot marshal type: " + in.Type().String()) + } +} + +func (e *encoder) mapv(tag string, in reflect.Value) { + e.mappingv(tag, func() { + keys := keyList(in.MapKeys()) + sort.Sort(keys) + for _, k := range keys { + e.marshal("", k) + e.marshal("", in.MapIndex(k)) + } + }) +} + +func (e *encoder) itemsv(tag string, in reflect.Value) { + e.mappingv(tag, func() { + slice := in.Convert(reflect.TypeOf([]MapItem{})).Interface().([]MapItem) + for _, item := range slice { + e.marshal("", reflect.ValueOf(item.Key)) + e.marshal("", reflect.ValueOf(item.Value)) + } + }) +} + +func (e *encoder) structv(tag string, in reflect.Value) { + sinfo, err := getStructInfo(in.Type()) + if err != nil { + panic(err) + } + e.mappingv(tag, func() { + for _, info := range sinfo.FieldsList { + var value reflect.Value + if info.Inline == nil { + value = in.Field(info.Num) + } else { + value = in.FieldByIndex(info.Inline) + } + if info.OmitEmpty && isZero(value) { + continue + } + e.marshal("", reflect.ValueOf(info.Key)) + e.flow = info.Flow + e.marshal("", value) + } + if sinfo.InlineMap >= 0 { + m := in.Field(sinfo.InlineMap) + if m.Len() > 0 { + e.flow = false + keys := keyList(m.MapKeys()) + sort.Sort(keys) + for _, k := range keys { + if _, found := sinfo.FieldsMap[k.String()]; found { + panic(fmt.Sprintf("Can't have key %q in inlined map; conflicts with struct field", k.String())) + } + e.marshal("", k) + e.flow = false + e.marshal("", m.MapIndex(k)) + } + } + } + }) +} + +func (e *encoder) mappingv(tag string, f func()) { + implicit := tag == "" + style := yaml_BLOCK_MAPPING_STYLE + if e.flow { + e.flow = false + style = yaml_FLOW_MAPPING_STYLE + } + yaml_mapping_start_event_initialize(&e.event, nil, []byte(tag), implicit, style) + e.emit() + f() + yaml_mapping_end_event_initialize(&e.event) + e.emit() +} + +func (e *encoder) slicev(tag string, in reflect.Value) { + implicit := tag == "" + style := yaml_BLOCK_SEQUENCE_STYLE + if e.flow { + e.flow = false + style = yaml_FLOW_SEQUENCE_STYLE + } + e.must(yaml_sequence_start_event_initialize(&e.event, nil, []byte(tag), implicit, style)) + e.emit() + n := in.Len() + for i := 0; i < n; i++ { + e.marshal("", in.Index(i)) + } + e.must(yaml_sequence_end_event_initialize(&e.event)) + e.emit() +} + +// isBase60 returns whether s is in base 60 notation as defined in YAML 1.1. +// +// The base 60 float notation in YAML 1.1 is a terrible idea and is unsupported +// in YAML 1.2 and by this package, but these should be marshalled quoted for +// the time being for compatibility with other parsers. +func isBase60Float(s string) (result bool) { + // Fast path. + if s == "" { + return false + } + c := s[0] + if !(c == '+' || c == '-' || c >= '0' && c <= '9') || strings.IndexByte(s, ':') < 0 { + return false + } + // Do the full match. + return base60float.MatchString(s) +} + +// From http://yaml.org/type/float.html, except the regular expression there +// is bogus. In practice parsers do not enforce the "\.[0-9_]*" suffix. +var base60float = regexp.MustCompile(`^[-+]?[0-9][0-9_]*(?::[0-5]?[0-9])+(?:\.[0-9_]*)?$`) + +func (e *encoder) stringv(tag string, in reflect.Value) { + var style yaml_scalar_style_t + s := in.String() + canUsePlain := true + switch { + case !utf8.ValidString(s): + if tag == yaml_BINARY_TAG { + failf("explicitly tagged !!binary data must be base64-encoded") + } + if tag != "" { + failf("cannot marshal invalid UTF-8 data as %s", shortTag(tag)) + } + // It can't be encoded directly as YAML so use a binary tag + // and encode it as base64. + tag = yaml_BINARY_TAG + s = encodeBase64(s) + case tag == "": + // Check to see if it would resolve to a specific + // tag when encoded unquoted. If it doesn't, + // there's no need to quote it. + rtag, _ := resolve("", s) + canUsePlain = rtag == yaml_STR_TAG && !isBase60Float(s) + } + // Note: it's possible for user code to emit invalid YAML + // if they explicitly specify a tag and a string containing + // text that's incompatible with that tag. + switch { + case strings.Contains(s, "\n"): + style = yaml_LITERAL_SCALAR_STYLE + case canUsePlain: + style = yaml_PLAIN_SCALAR_STYLE + default: + style = yaml_DOUBLE_QUOTED_SCALAR_STYLE + } + e.emitScalar(s, "", tag, style) +} + +func (e *encoder) boolv(tag string, in reflect.Value) { + var s string + if in.Bool() { + s = "true" + } else { + s = "false" + } + e.emitScalar(s, "", tag, yaml_PLAIN_SCALAR_STYLE) +} + +func (e *encoder) intv(tag string, in reflect.Value) { + s := strconv.FormatInt(in.Int(), 10) + e.emitScalar(s, "", tag, yaml_PLAIN_SCALAR_STYLE) +} + +func (e *encoder) uintv(tag string, in reflect.Value) { + s := strconv.FormatUint(in.Uint(), 10) + e.emitScalar(s, "", tag, yaml_PLAIN_SCALAR_STYLE) +} + +func (e *encoder) timev(tag string, in reflect.Value) { + t := in.Interface().(time.Time) + s := t.Format(time.RFC3339Nano) + e.emitScalar(s, "", tag, yaml_PLAIN_SCALAR_STYLE) +} + +func (e *encoder) floatv(tag string, in reflect.Value) { + // Issue #352: When formatting, use the precision of the underlying value + precision := 64 + if in.Kind() == reflect.Float32 { + precision = 32 + } + + s := strconv.FormatFloat(in.Float(), 'g', -1, precision) + switch s { + case "+Inf": + s = ".inf" + case "-Inf": + s = "-.inf" + case "NaN": + s = ".nan" + } + e.emitScalar(s, "", tag, yaml_PLAIN_SCALAR_STYLE) +} + +func (e *encoder) nilv() { + e.emitScalar("null", "", "", yaml_PLAIN_SCALAR_STYLE) +} + +func (e *encoder) emitScalar(value, anchor, tag string, style yaml_scalar_style_t) { + implicit := tag == "" + e.must(yaml_scalar_event_initialize(&e.event, []byte(anchor), []byte(tag), []byte(value), implicit, implicit, style)) + e.emit() +} diff --git a/tools/voluspa/vendor/gopkg.in/yaml.v2/parserc.go b/tools/voluspa/vendor/gopkg.in/yaml.v2/parserc.go new file mode 100644 index 00000000000..81d05dfe573 --- /dev/null +++ b/tools/voluspa/vendor/gopkg.in/yaml.v2/parserc.go @@ -0,0 +1,1095 @@ +package yaml + +import ( + "bytes" +) + +// The parser implements the following grammar: +// +// stream ::= STREAM-START implicit_document? explicit_document* STREAM-END +// implicit_document ::= block_node DOCUMENT-END* +// explicit_document ::= DIRECTIVE* DOCUMENT-START block_node? DOCUMENT-END* +// block_node_or_indentless_sequence ::= +// ALIAS +// | properties (block_content | indentless_block_sequence)? +// | block_content +// | indentless_block_sequence +// block_node ::= ALIAS +// | properties block_content? +// | block_content +// flow_node ::= ALIAS +// | properties flow_content? +// | flow_content +// properties ::= TAG ANCHOR? | ANCHOR TAG? +// block_content ::= block_collection | flow_collection | SCALAR +// flow_content ::= flow_collection | SCALAR +// block_collection ::= block_sequence | block_mapping +// flow_collection ::= flow_sequence | flow_mapping +// block_sequence ::= BLOCK-SEQUENCE-START (BLOCK-ENTRY block_node?)* BLOCK-END +// indentless_sequence ::= (BLOCK-ENTRY block_node?)+ +// block_mapping ::= BLOCK-MAPPING_START +// ((KEY block_node_or_indentless_sequence?)? +// (VALUE block_node_or_indentless_sequence?)?)* +// BLOCK-END +// flow_sequence ::= FLOW-SEQUENCE-START +// (flow_sequence_entry FLOW-ENTRY)* +// flow_sequence_entry? +// FLOW-SEQUENCE-END +// flow_sequence_entry ::= flow_node | KEY flow_node? (VALUE flow_node?)? +// flow_mapping ::= FLOW-MAPPING-START +// (flow_mapping_entry FLOW-ENTRY)* +// flow_mapping_entry? +// FLOW-MAPPING-END +// flow_mapping_entry ::= flow_node | KEY flow_node? (VALUE flow_node?)? + +// Peek the next token in the token queue. +func peek_token(parser *yaml_parser_t) *yaml_token_t { + if parser.token_available || yaml_parser_fetch_more_tokens(parser) { + return &parser.tokens[parser.tokens_head] + } + return nil +} + +// Remove the next token from the queue (must be called after peek_token). +func skip_token(parser *yaml_parser_t) { + parser.token_available = false + parser.tokens_parsed++ + parser.stream_end_produced = parser.tokens[parser.tokens_head].typ == yaml_STREAM_END_TOKEN + parser.tokens_head++ +} + +// Get the next event. +func yaml_parser_parse(parser *yaml_parser_t, event *yaml_event_t) bool { + // Erase the event object. + *event = yaml_event_t{} + + // No events after the end of the stream or error. + if parser.stream_end_produced || parser.error != yaml_NO_ERROR || parser.state == yaml_PARSE_END_STATE { + return true + } + + // Generate the next event. + return yaml_parser_state_machine(parser, event) +} + +// Set parser error. +func yaml_parser_set_parser_error(parser *yaml_parser_t, problem string, problem_mark yaml_mark_t) bool { + parser.error = yaml_PARSER_ERROR + parser.problem = problem + parser.problem_mark = problem_mark + return false +} + +func yaml_parser_set_parser_error_context(parser *yaml_parser_t, context string, context_mark yaml_mark_t, problem string, problem_mark yaml_mark_t) bool { + parser.error = yaml_PARSER_ERROR + parser.context = context + parser.context_mark = context_mark + parser.problem = problem + parser.problem_mark = problem_mark + return false +} + +// State dispatcher. +func yaml_parser_state_machine(parser *yaml_parser_t, event *yaml_event_t) bool { + //trace("yaml_parser_state_machine", "state:", parser.state.String()) + + switch parser.state { + case yaml_PARSE_STREAM_START_STATE: + return yaml_parser_parse_stream_start(parser, event) + + case yaml_PARSE_IMPLICIT_DOCUMENT_START_STATE: + return yaml_parser_parse_document_start(parser, event, true) + + case yaml_PARSE_DOCUMENT_START_STATE: + return yaml_parser_parse_document_start(parser, event, false) + + case yaml_PARSE_DOCUMENT_CONTENT_STATE: + return yaml_parser_parse_document_content(parser, event) + + case yaml_PARSE_DOCUMENT_END_STATE: + return yaml_parser_parse_document_end(parser, event) + + case yaml_PARSE_BLOCK_NODE_STATE: + return yaml_parser_parse_node(parser, event, true, false) + + case yaml_PARSE_BLOCK_NODE_OR_INDENTLESS_SEQUENCE_STATE: + return yaml_parser_parse_node(parser, event, true, true) + + case yaml_PARSE_FLOW_NODE_STATE: + return yaml_parser_parse_node(parser, event, false, false) + + case yaml_PARSE_BLOCK_SEQUENCE_FIRST_ENTRY_STATE: + return yaml_parser_parse_block_sequence_entry(parser, event, true) + + case yaml_PARSE_BLOCK_SEQUENCE_ENTRY_STATE: + return yaml_parser_parse_block_sequence_entry(parser, event, false) + + case yaml_PARSE_INDENTLESS_SEQUENCE_ENTRY_STATE: + return yaml_parser_parse_indentless_sequence_entry(parser, event) + + case yaml_PARSE_BLOCK_MAPPING_FIRST_KEY_STATE: + return yaml_parser_parse_block_mapping_key(parser, event, true) + + case yaml_PARSE_BLOCK_MAPPING_KEY_STATE: + return yaml_parser_parse_block_mapping_key(parser, event, false) + + case yaml_PARSE_BLOCK_MAPPING_VALUE_STATE: + return yaml_parser_parse_block_mapping_value(parser, event) + + case yaml_PARSE_FLOW_SEQUENCE_FIRST_ENTRY_STATE: + return yaml_parser_parse_flow_sequence_entry(parser, event, true) + + case yaml_PARSE_FLOW_SEQUENCE_ENTRY_STATE: + return yaml_parser_parse_flow_sequence_entry(parser, event, false) + + case yaml_PARSE_FLOW_SEQUENCE_ENTRY_MAPPING_KEY_STATE: + return yaml_parser_parse_flow_sequence_entry_mapping_key(parser, event) + + case yaml_PARSE_FLOW_SEQUENCE_ENTRY_MAPPING_VALUE_STATE: + return yaml_parser_parse_flow_sequence_entry_mapping_value(parser, event) + + case yaml_PARSE_FLOW_SEQUENCE_ENTRY_MAPPING_END_STATE: + return yaml_parser_parse_flow_sequence_entry_mapping_end(parser, event) + + case yaml_PARSE_FLOW_MAPPING_FIRST_KEY_STATE: + return yaml_parser_parse_flow_mapping_key(parser, event, true) + + case yaml_PARSE_FLOW_MAPPING_KEY_STATE: + return yaml_parser_parse_flow_mapping_key(parser, event, false) + + case yaml_PARSE_FLOW_MAPPING_VALUE_STATE: + return yaml_parser_parse_flow_mapping_value(parser, event, false) + + case yaml_PARSE_FLOW_MAPPING_EMPTY_VALUE_STATE: + return yaml_parser_parse_flow_mapping_value(parser, event, true) + + default: + panic("invalid parser state") + } +} + +// Parse the production: +// stream ::= STREAM-START implicit_document? explicit_document* STREAM-END +// ************ +func yaml_parser_parse_stream_start(parser *yaml_parser_t, event *yaml_event_t) bool { + token := peek_token(parser) + if token == nil { + return false + } + if token.typ != yaml_STREAM_START_TOKEN { + return yaml_parser_set_parser_error(parser, "did not find expected ", token.start_mark) + } + parser.state = yaml_PARSE_IMPLICIT_DOCUMENT_START_STATE + *event = yaml_event_t{ + typ: yaml_STREAM_START_EVENT, + start_mark: token.start_mark, + end_mark: token.end_mark, + encoding: token.encoding, + } + skip_token(parser) + return true +} + +// Parse the productions: +// implicit_document ::= block_node DOCUMENT-END* +// * +// explicit_document ::= DIRECTIVE* DOCUMENT-START block_node? DOCUMENT-END* +// ************************* +func yaml_parser_parse_document_start(parser *yaml_parser_t, event *yaml_event_t, implicit bool) bool { + + token := peek_token(parser) + if token == nil { + return false + } + + // Parse extra document end indicators. + if !implicit { + for token.typ == yaml_DOCUMENT_END_TOKEN { + skip_token(parser) + token = peek_token(parser) + if token == nil { + return false + } + } + } + + if implicit && token.typ != yaml_VERSION_DIRECTIVE_TOKEN && + token.typ != yaml_TAG_DIRECTIVE_TOKEN && + token.typ != yaml_DOCUMENT_START_TOKEN && + token.typ != yaml_STREAM_END_TOKEN { + // Parse an implicit document. + if !yaml_parser_process_directives(parser, nil, nil) { + return false + } + parser.states = append(parser.states, yaml_PARSE_DOCUMENT_END_STATE) + parser.state = yaml_PARSE_BLOCK_NODE_STATE + + *event = yaml_event_t{ + typ: yaml_DOCUMENT_START_EVENT, + start_mark: token.start_mark, + end_mark: token.end_mark, + } + + } else if token.typ != yaml_STREAM_END_TOKEN { + // Parse an explicit document. + var version_directive *yaml_version_directive_t + var tag_directives []yaml_tag_directive_t + start_mark := token.start_mark + if !yaml_parser_process_directives(parser, &version_directive, &tag_directives) { + return false + } + token = peek_token(parser) + if token == nil { + return false + } + if token.typ != yaml_DOCUMENT_START_TOKEN { + yaml_parser_set_parser_error(parser, + "did not find expected ", token.start_mark) + return false + } + parser.states = append(parser.states, yaml_PARSE_DOCUMENT_END_STATE) + parser.state = yaml_PARSE_DOCUMENT_CONTENT_STATE + end_mark := token.end_mark + + *event = yaml_event_t{ + typ: yaml_DOCUMENT_START_EVENT, + start_mark: start_mark, + end_mark: end_mark, + version_directive: version_directive, + tag_directives: tag_directives, + implicit: false, + } + skip_token(parser) + + } else { + // Parse the stream end. + parser.state = yaml_PARSE_END_STATE + *event = yaml_event_t{ + typ: yaml_STREAM_END_EVENT, + start_mark: token.start_mark, + end_mark: token.end_mark, + } + skip_token(parser) + } + + return true +} + +// Parse the productions: +// explicit_document ::= DIRECTIVE* DOCUMENT-START block_node? DOCUMENT-END* +// *********** +// +func yaml_parser_parse_document_content(parser *yaml_parser_t, event *yaml_event_t) bool { + token := peek_token(parser) + if token == nil { + return false + } + if token.typ == yaml_VERSION_DIRECTIVE_TOKEN || + token.typ == yaml_TAG_DIRECTIVE_TOKEN || + token.typ == yaml_DOCUMENT_START_TOKEN || + token.typ == yaml_DOCUMENT_END_TOKEN || + token.typ == yaml_STREAM_END_TOKEN { + parser.state = parser.states[len(parser.states)-1] + parser.states = parser.states[:len(parser.states)-1] + return yaml_parser_process_empty_scalar(parser, event, + token.start_mark) + } + return yaml_parser_parse_node(parser, event, true, false) +} + +// Parse the productions: +// implicit_document ::= block_node DOCUMENT-END* +// ************* +// explicit_document ::= DIRECTIVE* DOCUMENT-START block_node? DOCUMENT-END* +// +func yaml_parser_parse_document_end(parser *yaml_parser_t, event *yaml_event_t) bool { + token := peek_token(parser) + if token == nil { + return false + } + + start_mark := token.start_mark + end_mark := token.start_mark + + implicit := true + if token.typ == yaml_DOCUMENT_END_TOKEN { + end_mark = token.end_mark + skip_token(parser) + implicit = false + } + + parser.tag_directives = parser.tag_directives[:0] + + parser.state = yaml_PARSE_DOCUMENT_START_STATE + *event = yaml_event_t{ + typ: yaml_DOCUMENT_END_EVENT, + start_mark: start_mark, + end_mark: end_mark, + implicit: implicit, + } + return true +} + +// Parse the productions: +// block_node_or_indentless_sequence ::= +// ALIAS +// ***** +// | properties (block_content | indentless_block_sequence)? +// ********** * +// | block_content | indentless_block_sequence +// * +// block_node ::= ALIAS +// ***** +// | properties block_content? +// ********** * +// | block_content +// * +// flow_node ::= ALIAS +// ***** +// | properties flow_content? +// ********** * +// | flow_content +// * +// properties ::= TAG ANCHOR? | ANCHOR TAG? +// ************************* +// block_content ::= block_collection | flow_collection | SCALAR +// ****** +// flow_content ::= flow_collection | SCALAR +// ****** +func yaml_parser_parse_node(parser *yaml_parser_t, event *yaml_event_t, block, indentless_sequence bool) bool { + //defer trace("yaml_parser_parse_node", "block:", block, "indentless_sequence:", indentless_sequence)() + + token := peek_token(parser) + if token == nil { + return false + } + + if token.typ == yaml_ALIAS_TOKEN { + parser.state = parser.states[len(parser.states)-1] + parser.states = parser.states[:len(parser.states)-1] + *event = yaml_event_t{ + typ: yaml_ALIAS_EVENT, + start_mark: token.start_mark, + end_mark: token.end_mark, + anchor: token.value, + } + skip_token(parser) + return true + } + + start_mark := token.start_mark + end_mark := token.start_mark + + var tag_token bool + var tag_handle, tag_suffix, anchor []byte + var tag_mark yaml_mark_t + if token.typ == yaml_ANCHOR_TOKEN { + anchor = token.value + start_mark = token.start_mark + end_mark = token.end_mark + skip_token(parser) + token = peek_token(parser) + if token == nil { + return false + } + if token.typ == yaml_TAG_TOKEN { + tag_token = true + tag_handle = token.value + tag_suffix = token.suffix + tag_mark = token.start_mark + end_mark = token.end_mark + skip_token(parser) + token = peek_token(parser) + if token == nil { + return false + } + } + } else if token.typ == yaml_TAG_TOKEN { + tag_token = true + tag_handle = token.value + tag_suffix = token.suffix + start_mark = token.start_mark + tag_mark = token.start_mark + end_mark = token.end_mark + skip_token(parser) + token = peek_token(parser) + if token == nil { + return false + } + if token.typ == yaml_ANCHOR_TOKEN { + anchor = token.value + end_mark = token.end_mark + skip_token(parser) + token = peek_token(parser) + if token == nil { + return false + } + } + } + + var tag []byte + if tag_token { + if len(tag_handle) == 0 { + tag = tag_suffix + tag_suffix = nil + } else { + for i := range parser.tag_directives { + if bytes.Equal(parser.tag_directives[i].handle, tag_handle) { + tag = append([]byte(nil), parser.tag_directives[i].prefix...) + tag = append(tag, tag_suffix...) + break + } + } + if len(tag) == 0 { + yaml_parser_set_parser_error_context(parser, + "while parsing a node", start_mark, + "found undefined tag handle", tag_mark) + return false + } + } + } + + implicit := len(tag) == 0 + if indentless_sequence && token.typ == yaml_BLOCK_ENTRY_TOKEN { + end_mark = token.end_mark + parser.state = yaml_PARSE_INDENTLESS_SEQUENCE_ENTRY_STATE + *event = yaml_event_t{ + typ: yaml_SEQUENCE_START_EVENT, + start_mark: start_mark, + end_mark: end_mark, + anchor: anchor, + tag: tag, + implicit: implicit, + style: yaml_style_t(yaml_BLOCK_SEQUENCE_STYLE), + } + return true + } + if token.typ == yaml_SCALAR_TOKEN { + var plain_implicit, quoted_implicit bool + end_mark = token.end_mark + if (len(tag) == 0 && token.style == yaml_PLAIN_SCALAR_STYLE) || (len(tag) == 1 && tag[0] == '!') { + plain_implicit = true + } else if len(tag) == 0 { + quoted_implicit = true + } + parser.state = parser.states[len(parser.states)-1] + parser.states = parser.states[:len(parser.states)-1] + + *event = yaml_event_t{ + typ: yaml_SCALAR_EVENT, + start_mark: start_mark, + end_mark: end_mark, + anchor: anchor, + tag: tag, + value: token.value, + implicit: plain_implicit, + quoted_implicit: quoted_implicit, + style: yaml_style_t(token.style), + } + skip_token(parser) + return true + } + if token.typ == yaml_FLOW_SEQUENCE_START_TOKEN { + // [Go] Some of the events below can be merged as they differ only on style. + end_mark = token.end_mark + parser.state = yaml_PARSE_FLOW_SEQUENCE_FIRST_ENTRY_STATE + *event = yaml_event_t{ + typ: yaml_SEQUENCE_START_EVENT, + start_mark: start_mark, + end_mark: end_mark, + anchor: anchor, + tag: tag, + implicit: implicit, + style: yaml_style_t(yaml_FLOW_SEQUENCE_STYLE), + } + return true + } + if token.typ == yaml_FLOW_MAPPING_START_TOKEN { + end_mark = token.end_mark + parser.state = yaml_PARSE_FLOW_MAPPING_FIRST_KEY_STATE + *event = yaml_event_t{ + typ: yaml_MAPPING_START_EVENT, + start_mark: start_mark, + end_mark: end_mark, + anchor: anchor, + tag: tag, + implicit: implicit, + style: yaml_style_t(yaml_FLOW_MAPPING_STYLE), + } + return true + } + if block && token.typ == yaml_BLOCK_SEQUENCE_START_TOKEN { + end_mark = token.end_mark + parser.state = yaml_PARSE_BLOCK_SEQUENCE_FIRST_ENTRY_STATE + *event = yaml_event_t{ + typ: yaml_SEQUENCE_START_EVENT, + start_mark: start_mark, + end_mark: end_mark, + anchor: anchor, + tag: tag, + implicit: implicit, + style: yaml_style_t(yaml_BLOCK_SEQUENCE_STYLE), + } + return true + } + if block && token.typ == yaml_BLOCK_MAPPING_START_TOKEN { + end_mark = token.end_mark + parser.state = yaml_PARSE_BLOCK_MAPPING_FIRST_KEY_STATE + *event = yaml_event_t{ + typ: yaml_MAPPING_START_EVENT, + start_mark: start_mark, + end_mark: end_mark, + anchor: anchor, + tag: tag, + implicit: implicit, + style: yaml_style_t(yaml_BLOCK_MAPPING_STYLE), + } + return true + } + if len(anchor) > 0 || len(tag) > 0 { + parser.state = parser.states[len(parser.states)-1] + parser.states = parser.states[:len(parser.states)-1] + + *event = yaml_event_t{ + typ: yaml_SCALAR_EVENT, + start_mark: start_mark, + end_mark: end_mark, + anchor: anchor, + tag: tag, + implicit: implicit, + quoted_implicit: false, + style: yaml_style_t(yaml_PLAIN_SCALAR_STYLE), + } + return true + } + + context := "while parsing a flow node" + if block { + context = "while parsing a block node" + } + yaml_parser_set_parser_error_context(parser, context, start_mark, + "did not find expected node content", token.start_mark) + return false +} + +// Parse the productions: +// block_sequence ::= BLOCK-SEQUENCE-START (BLOCK-ENTRY block_node?)* BLOCK-END +// ******************** *********** * ********* +// +func yaml_parser_parse_block_sequence_entry(parser *yaml_parser_t, event *yaml_event_t, first bool) bool { + if first { + token := peek_token(parser) + parser.marks = append(parser.marks, token.start_mark) + skip_token(parser) + } + + token := peek_token(parser) + if token == nil { + return false + } + + if token.typ == yaml_BLOCK_ENTRY_TOKEN { + mark := token.end_mark + skip_token(parser) + token = peek_token(parser) + if token == nil { + return false + } + if token.typ != yaml_BLOCK_ENTRY_TOKEN && token.typ != yaml_BLOCK_END_TOKEN { + parser.states = append(parser.states, yaml_PARSE_BLOCK_SEQUENCE_ENTRY_STATE) + return yaml_parser_parse_node(parser, event, true, false) + } else { + parser.state = yaml_PARSE_BLOCK_SEQUENCE_ENTRY_STATE + return yaml_parser_process_empty_scalar(parser, event, mark) + } + } + if token.typ == yaml_BLOCK_END_TOKEN { + parser.state = parser.states[len(parser.states)-1] + parser.states = parser.states[:len(parser.states)-1] + parser.marks = parser.marks[:len(parser.marks)-1] + + *event = yaml_event_t{ + typ: yaml_SEQUENCE_END_EVENT, + start_mark: token.start_mark, + end_mark: token.end_mark, + } + + skip_token(parser) + return true + } + + context_mark := parser.marks[len(parser.marks)-1] + parser.marks = parser.marks[:len(parser.marks)-1] + return yaml_parser_set_parser_error_context(parser, + "while parsing a block collection", context_mark, + "did not find expected '-' indicator", token.start_mark) +} + +// Parse the productions: +// indentless_sequence ::= (BLOCK-ENTRY block_node?)+ +// *********** * +func yaml_parser_parse_indentless_sequence_entry(parser *yaml_parser_t, event *yaml_event_t) bool { + token := peek_token(parser) + if token == nil { + return false + } + + if token.typ == yaml_BLOCK_ENTRY_TOKEN { + mark := token.end_mark + skip_token(parser) + token = peek_token(parser) + if token == nil { + return false + } + if token.typ != yaml_BLOCK_ENTRY_TOKEN && + token.typ != yaml_KEY_TOKEN && + token.typ != yaml_VALUE_TOKEN && + token.typ != yaml_BLOCK_END_TOKEN { + parser.states = append(parser.states, yaml_PARSE_INDENTLESS_SEQUENCE_ENTRY_STATE) + return yaml_parser_parse_node(parser, event, true, false) + } + parser.state = yaml_PARSE_INDENTLESS_SEQUENCE_ENTRY_STATE + return yaml_parser_process_empty_scalar(parser, event, mark) + } + parser.state = parser.states[len(parser.states)-1] + parser.states = parser.states[:len(parser.states)-1] + + *event = yaml_event_t{ + typ: yaml_SEQUENCE_END_EVENT, + start_mark: token.start_mark, + end_mark: token.start_mark, // [Go] Shouldn't this be token.end_mark? + } + return true +} + +// Parse the productions: +// block_mapping ::= BLOCK-MAPPING_START +// ******************* +// ((KEY block_node_or_indentless_sequence?)? +// *** * +// (VALUE block_node_or_indentless_sequence?)?)* +// +// BLOCK-END +// ********* +// +func yaml_parser_parse_block_mapping_key(parser *yaml_parser_t, event *yaml_event_t, first bool) bool { + if first { + token := peek_token(parser) + parser.marks = append(parser.marks, token.start_mark) + skip_token(parser) + } + + token := peek_token(parser) + if token == nil { + return false + } + + if token.typ == yaml_KEY_TOKEN { + mark := token.end_mark + skip_token(parser) + token = peek_token(parser) + if token == nil { + return false + } + if token.typ != yaml_KEY_TOKEN && + token.typ != yaml_VALUE_TOKEN && + token.typ != yaml_BLOCK_END_TOKEN { + parser.states = append(parser.states, yaml_PARSE_BLOCK_MAPPING_VALUE_STATE) + return yaml_parser_parse_node(parser, event, true, true) + } else { + parser.state = yaml_PARSE_BLOCK_MAPPING_VALUE_STATE + return yaml_parser_process_empty_scalar(parser, event, mark) + } + } else if token.typ == yaml_BLOCK_END_TOKEN { + parser.state = parser.states[len(parser.states)-1] + parser.states = parser.states[:len(parser.states)-1] + parser.marks = parser.marks[:len(parser.marks)-1] + *event = yaml_event_t{ + typ: yaml_MAPPING_END_EVENT, + start_mark: token.start_mark, + end_mark: token.end_mark, + } + skip_token(parser) + return true + } + + context_mark := parser.marks[len(parser.marks)-1] + parser.marks = parser.marks[:len(parser.marks)-1] + return yaml_parser_set_parser_error_context(parser, + "while parsing a block mapping", context_mark, + "did not find expected key", token.start_mark) +} + +// Parse the productions: +// block_mapping ::= BLOCK-MAPPING_START +// +// ((KEY block_node_or_indentless_sequence?)? +// +// (VALUE block_node_or_indentless_sequence?)?)* +// ***** * +// BLOCK-END +// +// +func yaml_parser_parse_block_mapping_value(parser *yaml_parser_t, event *yaml_event_t) bool { + token := peek_token(parser) + if token == nil { + return false + } + if token.typ == yaml_VALUE_TOKEN { + mark := token.end_mark + skip_token(parser) + token = peek_token(parser) + if token == nil { + return false + } + if token.typ != yaml_KEY_TOKEN && + token.typ != yaml_VALUE_TOKEN && + token.typ != yaml_BLOCK_END_TOKEN { + parser.states = append(parser.states, yaml_PARSE_BLOCK_MAPPING_KEY_STATE) + return yaml_parser_parse_node(parser, event, true, true) + } + parser.state = yaml_PARSE_BLOCK_MAPPING_KEY_STATE + return yaml_parser_process_empty_scalar(parser, event, mark) + } + parser.state = yaml_PARSE_BLOCK_MAPPING_KEY_STATE + return yaml_parser_process_empty_scalar(parser, event, token.start_mark) +} + +// Parse the productions: +// flow_sequence ::= FLOW-SEQUENCE-START +// ******************* +// (flow_sequence_entry FLOW-ENTRY)* +// * ********** +// flow_sequence_entry? +// * +// FLOW-SEQUENCE-END +// ***************** +// flow_sequence_entry ::= flow_node | KEY flow_node? (VALUE flow_node?)? +// * +// +func yaml_parser_parse_flow_sequence_entry(parser *yaml_parser_t, event *yaml_event_t, first bool) bool { + if first { + token := peek_token(parser) + parser.marks = append(parser.marks, token.start_mark) + skip_token(parser) + } + token := peek_token(parser) + if token == nil { + return false + } + if token.typ != yaml_FLOW_SEQUENCE_END_TOKEN { + if !first { + if token.typ == yaml_FLOW_ENTRY_TOKEN { + skip_token(parser) + token = peek_token(parser) + if token == nil { + return false + } + } else { + context_mark := parser.marks[len(parser.marks)-1] + parser.marks = parser.marks[:len(parser.marks)-1] + return yaml_parser_set_parser_error_context(parser, + "while parsing a flow sequence", context_mark, + "did not find expected ',' or ']'", token.start_mark) + } + } + + if token.typ == yaml_KEY_TOKEN { + parser.state = yaml_PARSE_FLOW_SEQUENCE_ENTRY_MAPPING_KEY_STATE + *event = yaml_event_t{ + typ: yaml_MAPPING_START_EVENT, + start_mark: token.start_mark, + end_mark: token.end_mark, + implicit: true, + style: yaml_style_t(yaml_FLOW_MAPPING_STYLE), + } + skip_token(parser) + return true + } else if token.typ != yaml_FLOW_SEQUENCE_END_TOKEN { + parser.states = append(parser.states, yaml_PARSE_FLOW_SEQUENCE_ENTRY_STATE) + return yaml_parser_parse_node(parser, event, false, false) + } + } + + parser.state = parser.states[len(parser.states)-1] + parser.states = parser.states[:len(parser.states)-1] + parser.marks = parser.marks[:len(parser.marks)-1] + + *event = yaml_event_t{ + typ: yaml_SEQUENCE_END_EVENT, + start_mark: token.start_mark, + end_mark: token.end_mark, + } + + skip_token(parser) + return true +} + +// +// Parse the productions: +// flow_sequence_entry ::= flow_node | KEY flow_node? (VALUE flow_node?)? +// *** * +// +func yaml_parser_parse_flow_sequence_entry_mapping_key(parser *yaml_parser_t, event *yaml_event_t) bool { + token := peek_token(parser) + if token == nil { + return false + } + if token.typ != yaml_VALUE_TOKEN && + token.typ != yaml_FLOW_ENTRY_TOKEN && + token.typ != yaml_FLOW_SEQUENCE_END_TOKEN { + parser.states = append(parser.states, yaml_PARSE_FLOW_SEQUENCE_ENTRY_MAPPING_VALUE_STATE) + return yaml_parser_parse_node(parser, event, false, false) + } + mark := token.end_mark + skip_token(parser) + parser.state = yaml_PARSE_FLOW_SEQUENCE_ENTRY_MAPPING_VALUE_STATE + return yaml_parser_process_empty_scalar(parser, event, mark) +} + +// Parse the productions: +// flow_sequence_entry ::= flow_node | KEY flow_node? (VALUE flow_node?)? +// ***** * +// +func yaml_parser_parse_flow_sequence_entry_mapping_value(parser *yaml_parser_t, event *yaml_event_t) bool { + token := peek_token(parser) + if token == nil { + return false + } + if token.typ == yaml_VALUE_TOKEN { + skip_token(parser) + token := peek_token(parser) + if token == nil { + return false + } + if token.typ != yaml_FLOW_ENTRY_TOKEN && token.typ != yaml_FLOW_SEQUENCE_END_TOKEN { + parser.states = append(parser.states, yaml_PARSE_FLOW_SEQUENCE_ENTRY_MAPPING_END_STATE) + return yaml_parser_parse_node(parser, event, false, false) + } + } + parser.state = yaml_PARSE_FLOW_SEQUENCE_ENTRY_MAPPING_END_STATE + return yaml_parser_process_empty_scalar(parser, event, token.start_mark) +} + +// Parse the productions: +// flow_sequence_entry ::= flow_node | KEY flow_node? (VALUE flow_node?)? +// * +// +func yaml_parser_parse_flow_sequence_entry_mapping_end(parser *yaml_parser_t, event *yaml_event_t) bool { + token := peek_token(parser) + if token == nil { + return false + } + parser.state = yaml_PARSE_FLOW_SEQUENCE_ENTRY_STATE + *event = yaml_event_t{ + typ: yaml_MAPPING_END_EVENT, + start_mark: token.start_mark, + end_mark: token.start_mark, // [Go] Shouldn't this be end_mark? + } + return true +} + +// Parse the productions: +// flow_mapping ::= FLOW-MAPPING-START +// ****************** +// (flow_mapping_entry FLOW-ENTRY)* +// * ********** +// flow_mapping_entry? +// ****************** +// FLOW-MAPPING-END +// **************** +// flow_mapping_entry ::= flow_node | KEY flow_node? (VALUE flow_node?)? +// * *** * +// +func yaml_parser_parse_flow_mapping_key(parser *yaml_parser_t, event *yaml_event_t, first bool) bool { + if first { + token := peek_token(parser) + parser.marks = append(parser.marks, token.start_mark) + skip_token(parser) + } + + token := peek_token(parser) + if token == nil { + return false + } + + if token.typ != yaml_FLOW_MAPPING_END_TOKEN { + if !first { + if token.typ == yaml_FLOW_ENTRY_TOKEN { + skip_token(parser) + token = peek_token(parser) + if token == nil { + return false + } + } else { + context_mark := parser.marks[len(parser.marks)-1] + parser.marks = parser.marks[:len(parser.marks)-1] + return yaml_parser_set_parser_error_context(parser, + "while parsing a flow mapping", context_mark, + "did not find expected ',' or '}'", token.start_mark) + } + } + + if token.typ == yaml_KEY_TOKEN { + skip_token(parser) + token = peek_token(parser) + if token == nil { + return false + } + if token.typ != yaml_VALUE_TOKEN && + token.typ != yaml_FLOW_ENTRY_TOKEN && + token.typ != yaml_FLOW_MAPPING_END_TOKEN { + parser.states = append(parser.states, yaml_PARSE_FLOW_MAPPING_VALUE_STATE) + return yaml_parser_parse_node(parser, event, false, false) + } else { + parser.state = yaml_PARSE_FLOW_MAPPING_VALUE_STATE + return yaml_parser_process_empty_scalar(parser, event, token.start_mark) + } + } else if token.typ != yaml_FLOW_MAPPING_END_TOKEN { + parser.states = append(parser.states, yaml_PARSE_FLOW_MAPPING_EMPTY_VALUE_STATE) + return yaml_parser_parse_node(parser, event, false, false) + } + } + + parser.state = parser.states[len(parser.states)-1] + parser.states = parser.states[:len(parser.states)-1] + parser.marks = parser.marks[:len(parser.marks)-1] + *event = yaml_event_t{ + typ: yaml_MAPPING_END_EVENT, + start_mark: token.start_mark, + end_mark: token.end_mark, + } + skip_token(parser) + return true +} + +// Parse the productions: +// flow_mapping_entry ::= flow_node | KEY flow_node? (VALUE flow_node?)? +// * ***** * +// +func yaml_parser_parse_flow_mapping_value(parser *yaml_parser_t, event *yaml_event_t, empty bool) bool { + token := peek_token(parser) + if token == nil { + return false + } + if empty { + parser.state = yaml_PARSE_FLOW_MAPPING_KEY_STATE + return yaml_parser_process_empty_scalar(parser, event, token.start_mark) + } + if token.typ == yaml_VALUE_TOKEN { + skip_token(parser) + token = peek_token(parser) + if token == nil { + return false + } + if token.typ != yaml_FLOW_ENTRY_TOKEN && token.typ != yaml_FLOW_MAPPING_END_TOKEN { + parser.states = append(parser.states, yaml_PARSE_FLOW_MAPPING_KEY_STATE) + return yaml_parser_parse_node(parser, event, false, false) + } + } + parser.state = yaml_PARSE_FLOW_MAPPING_KEY_STATE + return yaml_parser_process_empty_scalar(parser, event, token.start_mark) +} + +// Generate an empty scalar event. +func yaml_parser_process_empty_scalar(parser *yaml_parser_t, event *yaml_event_t, mark yaml_mark_t) bool { + *event = yaml_event_t{ + typ: yaml_SCALAR_EVENT, + start_mark: mark, + end_mark: mark, + value: nil, // Empty + implicit: true, + style: yaml_style_t(yaml_PLAIN_SCALAR_STYLE), + } + return true +} + +var default_tag_directives = []yaml_tag_directive_t{ + {[]byte("!"), []byte("!")}, + {[]byte("!!"), []byte("tag:yaml.org,2002:")}, +} + +// Parse directives. +func yaml_parser_process_directives(parser *yaml_parser_t, + version_directive_ref **yaml_version_directive_t, + tag_directives_ref *[]yaml_tag_directive_t) bool { + + var version_directive *yaml_version_directive_t + var tag_directives []yaml_tag_directive_t + + token := peek_token(parser) + if token == nil { + return false + } + + for token.typ == yaml_VERSION_DIRECTIVE_TOKEN || token.typ == yaml_TAG_DIRECTIVE_TOKEN { + if token.typ == yaml_VERSION_DIRECTIVE_TOKEN { + if version_directive != nil { + yaml_parser_set_parser_error(parser, + "found duplicate %YAML directive", token.start_mark) + return false + } + if token.major != 1 || token.minor != 1 { + yaml_parser_set_parser_error(parser, + "found incompatible YAML document", token.start_mark) + return false + } + version_directive = &yaml_version_directive_t{ + major: token.major, + minor: token.minor, + } + } else if token.typ == yaml_TAG_DIRECTIVE_TOKEN { + value := yaml_tag_directive_t{ + handle: token.value, + prefix: token.prefix, + } + if !yaml_parser_append_tag_directive(parser, value, false, token.start_mark) { + return false + } + tag_directives = append(tag_directives, value) + } + + skip_token(parser) + token = peek_token(parser) + if token == nil { + return false + } + } + + for i := range default_tag_directives { + if !yaml_parser_append_tag_directive(parser, default_tag_directives[i], true, token.start_mark) { + return false + } + } + + if version_directive_ref != nil { + *version_directive_ref = version_directive + } + if tag_directives_ref != nil { + *tag_directives_ref = tag_directives + } + return true +} + +// Append a tag directive to the directives stack. +func yaml_parser_append_tag_directive(parser *yaml_parser_t, value yaml_tag_directive_t, allow_duplicates bool, mark yaml_mark_t) bool { + for i := range parser.tag_directives { + if bytes.Equal(value.handle, parser.tag_directives[i].handle) { + if allow_duplicates { + return true + } + return yaml_parser_set_parser_error(parser, "found duplicate %TAG directive", mark) + } + } + + // [Go] I suspect the copy is unnecessary. This was likely done + // because there was no way to track ownership of the data. + value_copy := yaml_tag_directive_t{ + handle: make([]byte, len(value.handle)), + prefix: make([]byte, len(value.prefix)), + } + copy(value_copy.handle, value.handle) + copy(value_copy.prefix, value.prefix) + parser.tag_directives = append(parser.tag_directives, value_copy) + return true +} diff --git a/tools/voluspa/vendor/gopkg.in/yaml.v2/readerc.go b/tools/voluspa/vendor/gopkg.in/yaml.v2/readerc.go new file mode 100644 index 00000000000..7c1f5fac3db --- /dev/null +++ b/tools/voluspa/vendor/gopkg.in/yaml.v2/readerc.go @@ -0,0 +1,412 @@ +package yaml + +import ( + "io" +) + +// Set the reader error and return 0. +func yaml_parser_set_reader_error(parser *yaml_parser_t, problem string, offset int, value int) bool { + parser.error = yaml_READER_ERROR + parser.problem = problem + parser.problem_offset = offset + parser.problem_value = value + return false +} + +// Byte order marks. +const ( + bom_UTF8 = "\xef\xbb\xbf" + bom_UTF16LE = "\xff\xfe" + bom_UTF16BE = "\xfe\xff" +) + +// Determine the input stream encoding by checking the BOM symbol. If no BOM is +// found, the UTF-8 encoding is assumed. Return 1 on success, 0 on failure. +func yaml_parser_determine_encoding(parser *yaml_parser_t) bool { + // Ensure that we had enough bytes in the raw buffer. + for !parser.eof && len(parser.raw_buffer)-parser.raw_buffer_pos < 3 { + if !yaml_parser_update_raw_buffer(parser) { + return false + } + } + + // Determine the encoding. + buf := parser.raw_buffer + pos := parser.raw_buffer_pos + avail := len(buf) - pos + if avail >= 2 && buf[pos] == bom_UTF16LE[0] && buf[pos+1] == bom_UTF16LE[1] { + parser.encoding = yaml_UTF16LE_ENCODING + parser.raw_buffer_pos += 2 + parser.offset += 2 + } else if avail >= 2 && buf[pos] == bom_UTF16BE[0] && buf[pos+1] == bom_UTF16BE[1] { + parser.encoding = yaml_UTF16BE_ENCODING + parser.raw_buffer_pos += 2 + parser.offset += 2 + } else if avail >= 3 && buf[pos] == bom_UTF8[0] && buf[pos+1] == bom_UTF8[1] && buf[pos+2] == bom_UTF8[2] { + parser.encoding = yaml_UTF8_ENCODING + parser.raw_buffer_pos += 3 + parser.offset += 3 + } else { + parser.encoding = yaml_UTF8_ENCODING + } + return true +} + +// Update the raw buffer. +func yaml_parser_update_raw_buffer(parser *yaml_parser_t) bool { + size_read := 0 + + // Return if the raw buffer is full. + if parser.raw_buffer_pos == 0 && len(parser.raw_buffer) == cap(parser.raw_buffer) { + return true + } + + // Return on EOF. + if parser.eof { + return true + } + + // Move the remaining bytes in the raw buffer to the beginning. + if parser.raw_buffer_pos > 0 && parser.raw_buffer_pos < len(parser.raw_buffer) { + copy(parser.raw_buffer, parser.raw_buffer[parser.raw_buffer_pos:]) + } + parser.raw_buffer = parser.raw_buffer[:len(parser.raw_buffer)-parser.raw_buffer_pos] + parser.raw_buffer_pos = 0 + + // Call the read handler to fill the buffer. + size_read, err := parser.read_handler(parser, parser.raw_buffer[len(parser.raw_buffer):cap(parser.raw_buffer)]) + parser.raw_buffer = parser.raw_buffer[:len(parser.raw_buffer)+size_read] + if err == io.EOF { + parser.eof = true + } else if err != nil { + return yaml_parser_set_reader_error(parser, "input error: "+err.Error(), parser.offset, -1) + } + return true +} + +// Ensure that the buffer contains at least `length` characters. +// Return true on success, false on failure. +// +// The length is supposed to be significantly less that the buffer size. +func yaml_parser_update_buffer(parser *yaml_parser_t, length int) bool { + if parser.read_handler == nil { + panic("read handler must be set") + } + + // [Go] This function was changed to guarantee the requested length size at EOF. + // The fact we need to do this is pretty awful, but the description above implies + // for that to be the case, and there are tests + + // If the EOF flag is set and the raw buffer is empty, do nothing. + if parser.eof && parser.raw_buffer_pos == len(parser.raw_buffer) { + // [Go] ACTUALLY! Read the documentation of this function above. + // This is just broken. To return true, we need to have the + // given length in the buffer. Not doing that means every single + // check that calls this function to make sure the buffer has a + // given length is Go) panicking; or C) accessing invalid memory. + //return true + } + + // Return if the buffer contains enough characters. + if parser.unread >= length { + return true + } + + // Determine the input encoding if it is not known yet. + if parser.encoding == yaml_ANY_ENCODING { + if !yaml_parser_determine_encoding(parser) { + return false + } + } + + // Move the unread characters to the beginning of the buffer. + buffer_len := len(parser.buffer) + if parser.buffer_pos > 0 && parser.buffer_pos < buffer_len { + copy(parser.buffer, parser.buffer[parser.buffer_pos:]) + buffer_len -= parser.buffer_pos + parser.buffer_pos = 0 + } else if parser.buffer_pos == buffer_len { + buffer_len = 0 + parser.buffer_pos = 0 + } + + // Open the whole buffer for writing, and cut it before returning. + parser.buffer = parser.buffer[:cap(parser.buffer)] + + // Fill the buffer until it has enough characters. + first := true + for parser.unread < length { + + // Fill the raw buffer if necessary. + if !first || parser.raw_buffer_pos == len(parser.raw_buffer) { + if !yaml_parser_update_raw_buffer(parser) { + parser.buffer = parser.buffer[:buffer_len] + return false + } + } + first = false + + // Decode the raw buffer. + inner: + for parser.raw_buffer_pos != len(parser.raw_buffer) { + var value rune + var width int + + raw_unread := len(parser.raw_buffer) - parser.raw_buffer_pos + + // Decode the next character. + switch parser.encoding { + case yaml_UTF8_ENCODING: + // Decode a UTF-8 character. Check RFC 3629 + // (http://www.ietf.org/rfc/rfc3629.txt) for more details. + // + // The following table (taken from the RFC) is used for + // decoding. + // + // Char. number range | UTF-8 octet sequence + // (hexadecimal) | (binary) + // --------------------+------------------------------------ + // 0000 0000-0000 007F | 0xxxxxxx + // 0000 0080-0000 07FF | 110xxxxx 10xxxxxx + // 0000 0800-0000 FFFF | 1110xxxx 10xxxxxx 10xxxxxx + // 0001 0000-0010 FFFF | 11110xxx 10xxxxxx 10xxxxxx 10xxxxxx + // + // Additionally, the characters in the range 0xD800-0xDFFF + // are prohibited as they are reserved for use with UTF-16 + // surrogate pairs. + + // Determine the length of the UTF-8 sequence. + octet := parser.raw_buffer[parser.raw_buffer_pos] + switch { + case octet&0x80 == 0x00: + width = 1 + case octet&0xE0 == 0xC0: + width = 2 + case octet&0xF0 == 0xE0: + width = 3 + case octet&0xF8 == 0xF0: + width = 4 + default: + // The leading octet is invalid. + return yaml_parser_set_reader_error(parser, + "invalid leading UTF-8 octet", + parser.offset, int(octet)) + } + + // Check if the raw buffer contains an incomplete character. + if width > raw_unread { + if parser.eof { + return yaml_parser_set_reader_error(parser, + "incomplete UTF-8 octet sequence", + parser.offset, -1) + } + break inner + } + + // Decode the leading octet. + switch { + case octet&0x80 == 0x00: + value = rune(octet & 0x7F) + case octet&0xE0 == 0xC0: + value = rune(octet & 0x1F) + case octet&0xF0 == 0xE0: + value = rune(octet & 0x0F) + case octet&0xF8 == 0xF0: + value = rune(octet & 0x07) + default: + value = 0 + } + + // Check and decode the trailing octets. + for k := 1; k < width; k++ { + octet = parser.raw_buffer[parser.raw_buffer_pos+k] + + // Check if the octet is valid. + if (octet & 0xC0) != 0x80 { + return yaml_parser_set_reader_error(parser, + "invalid trailing UTF-8 octet", + parser.offset+k, int(octet)) + } + + // Decode the octet. + value = (value << 6) + rune(octet&0x3F) + } + + // Check the length of the sequence against the value. + switch { + case width == 1: + case width == 2 && value >= 0x80: + case width == 3 && value >= 0x800: + case width == 4 && value >= 0x10000: + default: + return yaml_parser_set_reader_error(parser, + "invalid length of a UTF-8 sequence", + parser.offset, -1) + } + + // Check the range of the value. + if value >= 0xD800 && value <= 0xDFFF || value > 0x10FFFF { + return yaml_parser_set_reader_error(parser, + "invalid Unicode character", + parser.offset, int(value)) + } + + case yaml_UTF16LE_ENCODING, yaml_UTF16BE_ENCODING: + var low, high int + if parser.encoding == yaml_UTF16LE_ENCODING { + low, high = 0, 1 + } else { + low, high = 1, 0 + } + + // The UTF-16 encoding is not as simple as one might + // naively think. Check RFC 2781 + // (http://www.ietf.org/rfc/rfc2781.txt). + // + // Normally, two subsequent bytes describe a Unicode + // character. However a special technique (called a + // surrogate pair) is used for specifying character + // values larger than 0xFFFF. + // + // A surrogate pair consists of two pseudo-characters: + // high surrogate area (0xD800-0xDBFF) + // low surrogate area (0xDC00-0xDFFF) + // + // The following formulas are used for decoding + // and encoding characters using surrogate pairs: + // + // U = U' + 0x10000 (0x01 00 00 <= U <= 0x10 FF FF) + // U' = yyyyyyyyyyxxxxxxxxxx (0 <= U' <= 0x0F FF FF) + // W1 = 110110yyyyyyyyyy + // W2 = 110111xxxxxxxxxx + // + // where U is the character value, W1 is the high surrogate + // area, W2 is the low surrogate area. + + // Check for incomplete UTF-16 character. + if raw_unread < 2 { + if parser.eof { + return yaml_parser_set_reader_error(parser, + "incomplete UTF-16 character", + parser.offset, -1) + } + break inner + } + + // Get the character. + value = rune(parser.raw_buffer[parser.raw_buffer_pos+low]) + + (rune(parser.raw_buffer[parser.raw_buffer_pos+high]) << 8) + + // Check for unexpected low surrogate area. + if value&0xFC00 == 0xDC00 { + return yaml_parser_set_reader_error(parser, + "unexpected low surrogate area", + parser.offset, int(value)) + } + + // Check for a high surrogate area. + if value&0xFC00 == 0xD800 { + width = 4 + + // Check for incomplete surrogate pair. + if raw_unread < 4 { + if parser.eof { + return yaml_parser_set_reader_error(parser, + "incomplete UTF-16 surrogate pair", + parser.offset, -1) + } + break inner + } + + // Get the next character. + value2 := rune(parser.raw_buffer[parser.raw_buffer_pos+low+2]) + + (rune(parser.raw_buffer[parser.raw_buffer_pos+high+2]) << 8) + + // Check for a low surrogate area. + if value2&0xFC00 != 0xDC00 { + return yaml_parser_set_reader_error(parser, + "expected low surrogate area", + parser.offset+2, int(value2)) + } + + // Generate the value of the surrogate pair. + value = 0x10000 + ((value & 0x3FF) << 10) + (value2 & 0x3FF) + } else { + width = 2 + } + + default: + panic("impossible") + } + + // Check if the character is in the allowed range: + // #x9 | #xA | #xD | [#x20-#x7E] (8 bit) + // | #x85 | [#xA0-#xD7FF] | [#xE000-#xFFFD] (16 bit) + // | [#x10000-#x10FFFF] (32 bit) + switch { + case value == 0x09: + case value == 0x0A: + case value == 0x0D: + case value >= 0x20 && value <= 0x7E: + case value == 0x85: + case value >= 0xA0 && value <= 0xD7FF: + case value >= 0xE000 && value <= 0xFFFD: + case value >= 0x10000 && value <= 0x10FFFF: + default: + return yaml_parser_set_reader_error(parser, + "control characters are not allowed", + parser.offset, int(value)) + } + + // Move the raw pointers. + parser.raw_buffer_pos += width + parser.offset += width + + // Finally put the character into the buffer. + if value <= 0x7F { + // 0000 0000-0000 007F . 0xxxxxxx + parser.buffer[buffer_len+0] = byte(value) + buffer_len += 1 + } else if value <= 0x7FF { + // 0000 0080-0000 07FF . 110xxxxx 10xxxxxx + parser.buffer[buffer_len+0] = byte(0xC0 + (value >> 6)) + parser.buffer[buffer_len+1] = byte(0x80 + (value & 0x3F)) + buffer_len += 2 + } else if value <= 0xFFFF { + // 0000 0800-0000 FFFF . 1110xxxx 10xxxxxx 10xxxxxx + parser.buffer[buffer_len+0] = byte(0xE0 + (value >> 12)) + parser.buffer[buffer_len+1] = byte(0x80 + ((value >> 6) & 0x3F)) + parser.buffer[buffer_len+2] = byte(0x80 + (value & 0x3F)) + buffer_len += 3 + } else { + // 0001 0000-0010 FFFF . 11110xxx 10xxxxxx 10xxxxxx 10xxxxxx + parser.buffer[buffer_len+0] = byte(0xF0 + (value >> 18)) + parser.buffer[buffer_len+1] = byte(0x80 + ((value >> 12) & 0x3F)) + parser.buffer[buffer_len+2] = byte(0x80 + ((value >> 6) & 0x3F)) + parser.buffer[buffer_len+3] = byte(0x80 + (value & 0x3F)) + buffer_len += 4 + } + + parser.unread++ + } + + // On EOF, put NUL into the buffer and return. + if parser.eof { + parser.buffer[buffer_len] = 0 + buffer_len++ + parser.unread++ + break + } + } + // [Go] Read the documentation of this function above. To return true, + // we need to have the given length in the buffer. Not doing that means + // every single check that calls this function to make sure the buffer + // has a given length is Go) panicking; or C) accessing invalid memory. + // This happens here due to the EOF above breaking early. + for buffer_len < length { + parser.buffer[buffer_len] = 0 + buffer_len++ + } + parser.buffer = parser.buffer[:buffer_len] + return true +} diff --git a/tools/voluspa/vendor/gopkg.in/yaml.v2/resolve.go b/tools/voluspa/vendor/gopkg.in/yaml.v2/resolve.go new file mode 100644 index 00000000000..4120e0c9160 --- /dev/null +++ b/tools/voluspa/vendor/gopkg.in/yaml.v2/resolve.go @@ -0,0 +1,258 @@ +package yaml + +import ( + "encoding/base64" + "math" + "regexp" + "strconv" + "strings" + "time" +) + +type resolveMapItem struct { + value interface{} + tag string +} + +var resolveTable = make([]byte, 256) +var resolveMap = make(map[string]resolveMapItem) + +func init() { + t := resolveTable + t[int('+')] = 'S' // Sign + t[int('-')] = 'S' + for _, c := range "0123456789" { + t[int(c)] = 'D' // Digit + } + for _, c := range "yYnNtTfFoO~" { + t[int(c)] = 'M' // In map + } + t[int('.')] = '.' // Float (potentially in map) + + var resolveMapList = []struct { + v interface{} + tag string + l []string + }{ + {true, yaml_BOOL_TAG, []string{"y", "Y", "yes", "Yes", "YES"}}, + {true, yaml_BOOL_TAG, []string{"true", "True", "TRUE"}}, + {true, yaml_BOOL_TAG, []string{"on", "On", "ON"}}, + {false, yaml_BOOL_TAG, []string{"n", "N", "no", "No", "NO"}}, + {false, yaml_BOOL_TAG, []string{"false", "False", "FALSE"}}, + {false, yaml_BOOL_TAG, []string{"off", "Off", "OFF"}}, + {nil, yaml_NULL_TAG, []string{"", "~", "null", "Null", "NULL"}}, + {math.NaN(), yaml_FLOAT_TAG, []string{".nan", ".NaN", ".NAN"}}, + {math.Inf(+1), yaml_FLOAT_TAG, []string{".inf", ".Inf", ".INF"}}, + {math.Inf(+1), yaml_FLOAT_TAG, []string{"+.inf", "+.Inf", "+.INF"}}, + {math.Inf(-1), yaml_FLOAT_TAG, []string{"-.inf", "-.Inf", "-.INF"}}, + {"<<", yaml_MERGE_TAG, []string{"<<"}}, + } + + m := resolveMap + for _, item := range resolveMapList { + for _, s := range item.l { + m[s] = resolveMapItem{item.v, item.tag} + } + } +} + +const longTagPrefix = "tag:yaml.org,2002:" + +func shortTag(tag string) string { + // TODO This can easily be made faster and produce less garbage. + if strings.HasPrefix(tag, longTagPrefix) { + return "!!" + tag[len(longTagPrefix):] + } + return tag +} + +func longTag(tag string) string { + if strings.HasPrefix(tag, "!!") { + return longTagPrefix + tag[2:] + } + return tag +} + +func resolvableTag(tag string) bool { + switch tag { + case "", yaml_STR_TAG, yaml_BOOL_TAG, yaml_INT_TAG, yaml_FLOAT_TAG, yaml_NULL_TAG, yaml_TIMESTAMP_TAG: + return true + } + return false +} + +var yamlStyleFloat = regexp.MustCompile(`^[-+]?(\.[0-9]+|[0-9]+(\.[0-9]*)?)([eE][-+]?[0-9]+)?$`) + +func resolve(tag string, in string) (rtag string, out interface{}) { + if !resolvableTag(tag) { + return tag, in + } + + defer func() { + switch tag { + case "", rtag, yaml_STR_TAG, yaml_BINARY_TAG: + return + case yaml_FLOAT_TAG: + if rtag == yaml_INT_TAG { + switch v := out.(type) { + case int64: + rtag = yaml_FLOAT_TAG + out = float64(v) + return + case int: + rtag = yaml_FLOAT_TAG + out = float64(v) + return + } + } + } + failf("cannot decode %s `%s` as a %s", shortTag(rtag), in, shortTag(tag)) + }() + + // Any data is accepted as a !!str or !!binary. + // Otherwise, the prefix is enough of a hint about what it might be. + hint := byte('N') + if in != "" { + hint = resolveTable[in[0]] + } + if hint != 0 && tag != yaml_STR_TAG && tag != yaml_BINARY_TAG { + // Handle things we can lookup in a map. + if item, ok := resolveMap[in]; ok { + return item.tag, item.value + } + + // Base 60 floats are a bad idea, were dropped in YAML 1.2, and + // are purposefully unsupported here. They're still quoted on + // the way out for compatibility with other parser, though. + + switch hint { + case 'M': + // We've already checked the map above. + + case '.': + // Not in the map, so maybe a normal float. + floatv, err := strconv.ParseFloat(in, 64) + if err == nil { + return yaml_FLOAT_TAG, floatv + } + + case 'D', 'S': + // Int, float, or timestamp. + // Only try values as a timestamp if the value is unquoted or there's an explicit + // !!timestamp tag. + if tag == "" || tag == yaml_TIMESTAMP_TAG { + t, ok := parseTimestamp(in) + if ok { + return yaml_TIMESTAMP_TAG, t + } + } + + plain := strings.Replace(in, "_", "", -1) + intv, err := strconv.ParseInt(plain, 0, 64) + if err == nil { + if intv == int64(int(intv)) { + return yaml_INT_TAG, int(intv) + } else { + return yaml_INT_TAG, intv + } + } + uintv, err := strconv.ParseUint(plain, 0, 64) + if err == nil { + return yaml_INT_TAG, uintv + } + if yamlStyleFloat.MatchString(plain) { + floatv, err := strconv.ParseFloat(plain, 64) + if err == nil { + return yaml_FLOAT_TAG, floatv + } + } + if strings.HasPrefix(plain, "0b") { + intv, err := strconv.ParseInt(plain[2:], 2, 64) + if err == nil { + if intv == int64(int(intv)) { + return yaml_INT_TAG, int(intv) + } else { + return yaml_INT_TAG, intv + } + } + uintv, err := strconv.ParseUint(plain[2:], 2, 64) + if err == nil { + return yaml_INT_TAG, uintv + } + } else if strings.HasPrefix(plain, "-0b") { + intv, err := strconv.ParseInt("-" + plain[3:], 2, 64) + if err == nil { + if true || intv == int64(int(intv)) { + return yaml_INT_TAG, int(intv) + } else { + return yaml_INT_TAG, intv + } + } + } + default: + panic("resolveTable item not yet handled: " + string(rune(hint)) + " (with " + in + ")") + } + } + return yaml_STR_TAG, in +} + +// encodeBase64 encodes s as base64 that is broken up into multiple lines +// as appropriate for the resulting length. +func encodeBase64(s string) string { + const lineLen = 70 + encLen := base64.StdEncoding.EncodedLen(len(s)) + lines := encLen/lineLen + 1 + buf := make([]byte, encLen*2+lines) + in := buf[0:encLen] + out := buf[encLen:] + base64.StdEncoding.Encode(in, []byte(s)) + k := 0 + for i := 0; i < len(in); i += lineLen { + j := i + lineLen + if j > len(in) { + j = len(in) + } + k += copy(out[k:], in[i:j]) + if lines > 1 { + out[k] = '\n' + k++ + } + } + return string(out[:k]) +} + +// This is a subset of the formats allowed by the regular expression +// defined at http://yaml.org/type/timestamp.html. +var allowedTimestampFormats = []string{ + "2006-1-2T15:4:5.999999999Z07:00", // RCF3339Nano with short date fields. + "2006-1-2t15:4:5.999999999Z07:00", // RFC3339Nano with short date fields and lower-case "t". + "2006-1-2 15:4:5.999999999", // space separated with no time zone + "2006-1-2", // date only + // Notable exception: time.Parse cannot handle: "2001-12-14 21:59:43.10 -5" + // from the set of examples. +} + +// parseTimestamp parses s as a timestamp string and +// returns the timestamp and reports whether it succeeded. +// Timestamp formats are defined at http://yaml.org/type/timestamp.html +func parseTimestamp(s string) (time.Time, bool) { + // TODO write code to check all the formats supported by + // http://yaml.org/type/timestamp.html instead of using time.Parse. + + // Quick check: all date formats start with YYYY-. + i := 0 + for ; i < len(s); i++ { + if c := s[i]; c < '0' || c > '9' { + break + } + } + if i != 4 || i == len(s) || s[i] != '-' { + return time.Time{}, false + } + for _, format := range allowedTimestampFormats { + if t, err := time.Parse(format, s); err == nil { + return t, true + } + } + return time.Time{}, false +} diff --git a/tools/voluspa/vendor/gopkg.in/yaml.v2/scannerc.go b/tools/voluspa/vendor/gopkg.in/yaml.v2/scannerc.go new file mode 100644 index 00000000000..077fd1dd2d4 --- /dev/null +++ b/tools/voluspa/vendor/gopkg.in/yaml.v2/scannerc.go @@ -0,0 +1,2696 @@ +package yaml + +import ( + "bytes" + "fmt" +) + +// Introduction +// ************ +// +// The following notes assume that you are familiar with the YAML specification +// (http://yaml.org/spec/1.2/spec.html). We mostly follow it, although in +// some cases we are less restrictive that it requires. +// +// The process of transforming a YAML stream into a sequence of events is +// divided on two steps: Scanning and Parsing. +// +// The Scanner transforms the input stream into a sequence of tokens, while the +// parser transform the sequence of tokens produced by the Scanner into a +// sequence of parsing events. +// +// The Scanner is rather clever and complicated. The Parser, on the contrary, +// is a straightforward implementation of a recursive-descendant parser (or, +// LL(1) parser, as it is usually called). +// +// Actually there are two issues of Scanning that might be called "clever", the +// rest is quite straightforward. The issues are "block collection start" and +// "simple keys". Both issues are explained below in details. +// +// Here the Scanning step is explained and implemented. We start with the list +// of all the tokens produced by the Scanner together with short descriptions. +// +// Now, tokens: +// +// STREAM-START(encoding) # The stream start. +// STREAM-END # The stream end. +// VERSION-DIRECTIVE(major,minor) # The '%YAML' directive. +// TAG-DIRECTIVE(handle,prefix) # The '%TAG' directive. +// DOCUMENT-START # '---' +// DOCUMENT-END # '...' +// BLOCK-SEQUENCE-START # Indentation increase denoting a block +// BLOCK-MAPPING-START # sequence or a block mapping. +// BLOCK-END # Indentation decrease. +// FLOW-SEQUENCE-START # '[' +// FLOW-SEQUENCE-END # ']' +// BLOCK-SEQUENCE-START # '{' +// BLOCK-SEQUENCE-END # '}' +// BLOCK-ENTRY # '-' +// FLOW-ENTRY # ',' +// KEY # '?' or nothing (simple keys). +// VALUE # ':' +// ALIAS(anchor) # '*anchor' +// ANCHOR(anchor) # '&anchor' +// TAG(handle,suffix) # '!handle!suffix' +// SCALAR(value,style) # A scalar. +// +// The following two tokens are "virtual" tokens denoting the beginning and the +// end of the stream: +// +// STREAM-START(encoding) +// STREAM-END +// +// We pass the information about the input stream encoding with the +// STREAM-START token. +// +// The next two tokens are responsible for tags: +// +// VERSION-DIRECTIVE(major,minor) +// TAG-DIRECTIVE(handle,prefix) +// +// Example: +// +// %YAML 1.1 +// %TAG ! !foo +// %TAG !yaml! tag:yaml.org,2002: +// --- +// +// The correspoding sequence of tokens: +// +// STREAM-START(utf-8) +// VERSION-DIRECTIVE(1,1) +// TAG-DIRECTIVE("!","!foo") +// TAG-DIRECTIVE("!yaml","tag:yaml.org,2002:") +// DOCUMENT-START +// STREAM-END +// +// Note that the VERSION-DIRECTIVE and TAG-DIRECTIVE tokens occupy a whole +// line. +// +// The document start and end indicators are represented by: +// +// DOCUMENT-START +// DOCUMENT-END +// +// Note that if a YAML stream contains an implicit document (without '---' +// and '...' indicators), no DOCUMENT-START and DOCUMENT-END tokens will be +// produced. +// +// In the following examples, we present whole documents together with the +// produced tokens. +// +// 1. An implicit document: +// +// 'a scalar' +// +// Tokens: +// +// STREAM-START(utf-8) +// SCALAR("a scalar",single-quoted) +// STREAM-END +// +// 2. An explicit document: +// +// --- +// 'a scalar' +// ... +// +// Tokens: +// +// STREAM-START(utf-8) +// DOCUMENT-START +// SCALAR("a scalar",single-quoted) +// DOCUMENT-END +// STREAM-END +// +// 3. Several documents in a stream: +// +// 'a scalar' +// --- +// 'another scalar' +// --- +// 'yet another scalar' +// +// Tokens: +// +// STREAM-START(utf-8) +// SCALAR("a scalar",single-quoted) +// DOCUMENT-START +// SCALAR("another scalar",single-quoted) +// DOCUMENT-START +// SCALAR("yet another scalar",single-quoted) +// STREAM-END +// +// We have already introduced the SCALAR token above. The following tokens are +// used to describe aliases, anchors, tag, and scalars: +// +// ALIAS(anchor) +// ANCHOR(anchor) +// TAG(handle,suffix) +// SCALAR(value,style) +// +// The following series of examples illustrate the usage of these tokens: +// +// 1. A recursive sequence: +// +// &A [ *A ] +// +// Tokens: +// +// STREAM-START(utf-8) +// ANCHOR("A") +// FLOW-SEQUENCE-START +// ALIAS("A") +// FLOW-SEQUENCE-END +// STREAM-END +// +// 2. A tagged scalar: +// +// !!float "3.14" # A good approximation. +// +// Tokens: +// +// STREAM-START(utf-8) +// TAG("!!","float") +// SCALAR("3.14",double-quoted) +// STREAM-END +// +// 3. Various scalar styles: +// +// --- # Implicit empty plain scalars do not produce tokens. +// --- a plain scalar +// --- 'a single-quoted scalar' +// --- "a double-quoted scalar" +// --- |- +// a literal scalar +// --- >- +// a folded +// scalar +// +// Tokens: +// +// STREAM-START(utf-8) +// DOCUMENT-START +// DOCUMENT-START +// SCALAR("a plain scalar",plain) +// DOCUMENT-START +// SCALAR("a single-quoted scalar",single-quoted) +// DOCUMENT-START +// SCALAR("a double-quoted scalar",double-quoted) +// DOCUMENT-START +// SCALAR("a literal scalar",literal) +// DOCUMENT-START +// SCALAR("a folded scalar",folded) +// STREAM-END +// +// Now it's time to review collection-related tokens. We will start with +// flow collections: +// +// FLOW-SEQUENCE-START +// FLOW-SEQUENCE-END +// FLOW-MAPPING-START +// FLOW-MAPPING-END +// FLOW-ENTRY +// KEY +// VALUE +// +// The tokens FLOW-SEQUENCE-START, FLOW-SEQUENCE-END, FLOW-MAPPING-START, and +// FLOW-MAPPING-END represent the indicators '[', ']', '{', and '}' +// correspondingly. FLOW-ENTRY represent the ',' indicator. Finally the +// indicators '?' and ':', which are used for denoting mapping keys and values, +// are represented by the KEY and VALUE tokens. +// +// The following examples show flow collections: +// +// 1. A flow sequence: +// +// [item 1, item 2, item 3] +// +// Tokens: +// +// STREAM-START(utf-8) +// FLOW-SEQUENCE-START +// SCALAR("item 1",plain) +// FLOW-ENTRY +// SCALAR("item 2",plain) +// FLOW-ENTRY +// SCALAR("item 3",plain) +// FLOW-SEQUENCE-END +// STREAM-END +// +// 2. A flow mapping: +// +// { +// a simple key: a value, # Note that the KEY token is produced. +// ? a complex key: another value, +// } +// +// Tokens: +// +// STREAM-START(utf-8) +// FLOW-MAPPING-START +// KEY +// SCALAR("a simple key",plain) +// VALUE +// SCALAR("a value",plain) +// FLOW-ENTRY +// KEY +// SCALAR("a complex key",plain) +// VALUE +// SCALAR("another value",plain) +// FLOW-ENTRY +// FLOW-MAPPING-END +// STREAM-END +// +// A simple key is a key which is not denoted by the '?' indicator. Note that +// the Scanner still produce the KEY token whenever it encounters a simple key. +// +// For scanning block collections, the following tokens are used (note that we +// repeat KEY and VALUE here): +// +// BLOCK-SEQUENCE-START +// BLOCK-MAPPING-START +// BLOCK-END +// BLOCK-ENTRY +// KEY +// VALUE +// +// The tokens BLOCK-SEQUENCE-START and BLOCK-MAPPING-START denote indentation +// increase that precedes a block collection (cf. the INDENT token in Python). +// The token BLOCK-END denote indentation decrease that ends a block collection +// (cf. the DEDENT token in Python). However YAML has some syntax pecularities +// that makes detections of these tokens more complex. +// +// The tokens BLOCK-ENTRY, KEY, and VALUE are used to represent the indicators +// '-', '?', and ':' correspondingly. +// +// The following examples show how the tokens BLOCK-SEQUENCE-START, +// BLOCK-MAPPING-START, and BLOCK-END are emitted by the Scanner: +// +// 1. Block sequences: +// +// - item 1 +// - item 2 +// - +// - item 3.1 +// - item 3.2 +// - +// key 1: value 1 +// key 2: value 2 +// +// Tokens: +// +// STREAM-START(utf-8) +// BLOCK-SEQUENCE-START +// BLOCK-ENTRY +// SCALAR("item 1",plain) +// BLOCK-ENTRY +// SCALAR("item 2",plain) +// BLOCK-ENTRY +// BLOCK-SEQUENCE-START +// BLOCK-ENTRY +// SCALAR("item 3.1",plain) +// BLOCK-ENTRY +// SCALAR("item 3.2",plain) +// BLOCK-END +// BLOCK-ENTRY +// BLOCK-MAPPING-START +// KEY +// SCALAR("key 1",plain) +// VALUE +// SCALAR("value 1",plain) +// KEY +// SCALAR("key 2",plain) +// VALUE +// SCALAR("value 2",plain) +// BLOCK-END +// BLOCK-END +// STREAM-END +// +// 2. Block mappings: +// +// a simple key: a value # The KEY token is produced here. +// ? a complex key +// : another value +// a mapping: +// key 1: value 1 +// key 2: value 2 +// a sequence: +// - item 1 +// - item 2 +// +// Tokens: +// +// STREAM-START(utf-8) +// BLOCK-MAPPING-START +// KEY +// SCALAR("a simple key",plain) +// VALUE +// SCALAR("a value",plain) +// KEY +// SCALAR("a complex key",plain) +// VALUE +// SCALAR("another value",plain) +// KEY +// SCALAR("a mapping",plain) +// BLOCK-MAPPING-START +// KEY +// SCALAR("key 1",plain) +// VALUE +// SCALAR("value 1",plain) +// KEY +// SCALAR("key 2",plain) +// VALUE +// SCALAR("value 2",plain) +// BLOCK-END +// KEY +// SCALAR("a sequence",plain) +// VALUE +// BLOCK-SEQUENCE-START +// BLOCK-ENTRY +// SCALAR("item 1",plain) +// BLOCK-ENTRY +// SCALAR("item 2",plain) +// BLOCK-END +// BLOCK-END +// STREAM-END +// +// YAML does not always require to start a new block collection from a new +// line. If the current line contains only '-', '?', and ':' indicators, a new +// block collection may start at the current line. The following examples +// illustrate this case: +// +// 1. Collections in a sequence: +// +// - - item 1 +// - item 2 +// - key 1: value 1 +// key 2: value 2 +// - ? complex key +// : complex value +// +// Tokens: +// +// STREAM-START(utf-8) +// BLOCK-SEQUENCE-START +// BLOCK-ENTRY +// BLOCK-SEQUENCE-START +// BLOCK-ENTRY +// SCALAR("item 1",plain) +// BLOCK-ENTRY +// SCALAR("item 2",plain) +// BLOCK-END +// BLOCK-ENTRY +// BLOCK-MAPPING-START +// KEY +// SCALAR("key 1",plain) +// VALUE +// SCALAR("value 1",plain) +// KEY +// SCALAR("key 2",plain) +// VALUE +// SCALAR("value 2",plain) +// BLOCK-END +// BLOCK-ENTRY +// BLOCK-MAPPING-START +// KEY +// SCALAR("complex key") +// VALUE +// SCALAR("complex value") +// BLOCK-END +// BLOCK-END +// STREAM-END +// +// 2. Collections in a mapping: +// +// ? a sequence +// : - item 1 +// - item 2 +// ? a mapping +// : key 1: value 1 +// key 2: value 2 +// +// Tokens: +// +// STREAM-START(utf-8) +// BLOCK-MAPPING-START +// KEY +// SCALAR("a sequence",plain) +// VALUE +// BLOCK-SEQUENCE-START +// BLOCK-ENTRY +// SCALAR("item 1",plain) +// BLOCK-ENTRY +// SCALAR("item 2",plain) +// BLOCK-END +// KEY +// SCALAR("a mapping",plain) +// VALUE +// BLOCK-MAPPING-START +// KEY +// SCALAR("key 1",plain) +// VALUE +// SCALAR("value 1",plain) +// KEY +// SCALAR("key 2",plain) +// VALUE +// SCALAR("value 2",plain) +// BLOCK-END +// BLOCK-END +// STREAM-END +// +// YAML also permits non-indented sequences if they are included into a block +// mapping. In this case, the token BLOCK-SEQUENCE-START is not produced: +// +// key: +// - item 1 # BLOCK-SEQUENCE-START is NOT produced here. +// - item 2 +// +// Tokens: +// +// STREAM-START(utf-8) +// BLOCK-MAPPING-START +// KEY +// SCALAR("key",plain) +// VALUE +// BLOCK-ENTRY +// SCALAR("item 1",plain) +// BLOCK-ENTRY +// SCALAR("item 2",plain) +// BLOCK-END +// + +// Ensure that the buffer contains the required number of characters. +// Return true on success, false on failure (reader error or memory error). +func cache(parser *yaml_parser_t, length int) bool { + // [Go] This was inlined: !cache(A, B) -> unread < B && !update(A, B) + return parser.unread >= length || yaml_parser_update_buffer(parser, length) +} + +// Advance the buffer pointer. +func skip(parser *yaml_parser_t) { + parser.mark.index++ + parser.mark.column++ + parser.unread-- + parser.buffer_pos += width(parser.buffer[parser.buffer_pos]) +} + +func skip_line(parser *yaml_parser_t) { + if is_crlf(parser.buffer, parser.buffer_pos) { + parser.mark.index += 2 + parser.mark.column = 0 + parser.mark.line++ + parser.unread -= 2 + parser.buffer_pos += 2 + } else if is_break(parser.buffer, parser.buffer_pos) { + parser.mark.index++ + parser.mark.column = 0 + parser.mark.line++ + parser.unread-- + parser.buffer_pos += width(parser.buffer[parser.buffer_pos]) + } +} + +// Copy a character to a string buffer and advance pointers. +func read(parser *yaml_parser_t, s []byte) []byte { + w := width(parser.buffer[parser.buffer_pos]) + if w == 0 { + panic("invalid character sequence") + } + if len(s) == 0 { + s = make([]byte, 0, 32) + } + if w == 1 && len(s)+w <= cap(s) { + s = s[:len(s)+1] + s[len(s)-1] = parser.buffer[parser.buffer_pos] + parser.buffer_pos++ + } else { + s = append(s, parser.buffer[parser.buffer_pos:parser.buffer_pos+w]...) + parser.buffer_pos += w + } + parser.mark.index++ + parser.mark.column++ + parser.unread-- + return s +} + +// Copy a line break character to a string buffer and advance pointers. +func read_line(parser *yaml_parser_t, s []byte) []byte { + buf := parser.buffer + pos := parser.buffer_pos + switch { + case buf[pos] == '\r' && buf[pos+1] == '\n': + // CR LF . LF + s = append(s, '\n') + parser.buffer_pos += 2 + parser.mark.index++ + parser.unread-- + case buf[pos] == '\r' || buf[pos] == '\n': + // CR|LF . LF + s = append(s, '\n') + parser.buffer_pos += 1 + case buf[pos] == '\xC2' && buf[pos+1] == '\x85': + // NEL . LF + s = append(s, '\n') + parser.buffer_pos += 2 + case buf[pos] == '\xE2' && buf[pos+1] == '\x80' && (buf[pos+2] == '\xA8' || buf[pos+2] == '\xA9'): + // LS|PS . LS|PS + s = append(s, buf[parser.buffer_pos:pos+3]...) + parser.buffer_pos += 3 + default: + return s + } + parser.mark.index++ + parser.mark.column = 0 + parser.mark.line++ + parser.unread-- + return s +} + +// Get the next token. +func yaml_parser_scan(parser *yaml_parser_t, token *yaml_token_t) bool { + // Erase the token object. + *token = yaml_token_t{} // [Go] Is this necessary? + + // No tokens after STREAM-END or error. + if parser.stream_end_produced || parser.error != yaml_NO_ERROR { + return true + } + + // Ensure that the tokens queue contains enough tokens. + if !parser.token_available { + if !yaml_parser_fetch_more_tokens(parser) { + return false + } + } + + // Fetch the next token from the queue. + *token = parser.tokens[parser.tokens_head] + parser.tokens_head++ + parser.tokens_parsed++ + parser.token_available = false + + if token.typ == yaml_STREAM_END_TOKEN { + parser.stream_end_produced = true + } + return true +} + +// Set the scanner error and return false. +func yaml_parser_set_scanner_error(parser *yaml_parser_t, context string, context_mark yaml_mark_t, problem string) bool { + parser.error = yaml_SCANNER_ERROR + parser.context = context + parser.context_mark = context_mark + parser.problem = problem + parser.problem_mark = parser.mark + return false +} + +func yaml_parser_set_scanner_tag_error(parser *yaml_parser_t, directive bool, context_mark yaml_mark_t, problem string) bool { + context := "while parsing a tag" + if directive { + context = "while parsing a %TAG directive" + } + return yaml_parser_set_scanner_error(parser, context, context_mark, problem) +} + +func trace(args ...interface{}) func() { + pargs := append([]interface{}{"+++"}, args...) + fmt.Println(pargs...) + pargs = append([]interface{}{"---"}, args...) + return func() { fmt.Println(pargs...) } +} + +// Ensure that the tokens queue contains at least one token which can be +// returned to the Parser. +func yaml_parser_fetch_more_tokens(parser *yaml_parser_t) bool { + // While we need more tokens to fetch, do it. + for { + // Check if we really need to fetch more tokens. + need_more_tokens := false + + if parser.tokens_head == len(parser.tokens) { + // Queue is empty. + need_more_tokens = true + } else { + // Check if any potential simple key may occupy the head position. + if !yaml_parser_stale_simple_keys(parser) { + return false + } + + for i := range parser.simple_keys { + simple_key := &parser.simple_keys[i] + if simple_key.possible && simple_key.token_number == parser.tokens_parsed { + need_more_tokens = true + break + } + } + } + + // We are finished. + if !need_more_tokens { + break + } + // Fetch the next token. + if !yaml_parser_fetch_next_token(parser) { + return false + } + } + + parser.token_available = true + return true +} + +// The dispatcher for token fetchers. +func yaml_parser_fetch_next_token(parser *yaml_parser_t) bool { + // Ensure that the buffer is initialized. + if parser.unread < 1 && !yaml_parser_update_buffer(parser, 1) { + return false + } + + // Check if we just started scanning. Fetch STREAM-START then. + if !parser.stream_start_produced { + return yaml_parser_fetch_stream_start(parser) + } + + // Eat whitespaces and comments until we reach the next token. + if !yaml_parser_scan_to_next_token(parser) { + return false + } + + // Remove obsolete potential simple keys. + if !yaml_parser_stale_simple_keys(parser) { + return false + } + + // Check the indentation level against the current column. + if !yaml_parser_unroll_indent(parser, parser.mark.column) { + return false + } + + // Ensure that the buffer contains at least 4 characters. 4 is the length + // of the longest indicators ('--- ' and '... '). + if parser.unread < 4 && !yaml_parser_update_buffer(parser, 4) { + return false + } + + // Is it the end of the stream? + if is_z(parser.buffer, parser.buffer_pos) { + return yaml_parser_fetch_stream_end(parser) + } + + // Is it a directive? + if parser.mark.column == 0 && parser.buffer[parser.buffer_pos] == '%' { + return yaml_parser_fetch_directive(parser) + } + + buf := parser.buffer + pos := parser.buffer_pos + + // Is it the document start indicator? + if parser.mark.column == 0 && buf[pos] == '-' && buf[pos+1] == '-' && buf[pos+2] == '-' && is_blankz(buf, pos+3) { + return yaml_parser_fetch_document_indicator(parser, yaml_DOCUMENT_START_TOKEN) + } + + // Is it the document end indicator? + if parser.mark.column == 0 && buf[pos] == '.' && buf[pos+1] == '.' && buf[pos+2] == '.' && is_blankz(buf, pos+3) { + return yaml_parser_fetch_document_indicator(parser, yaml_DOCUMENT_END_TOKEN) + } + + // Is it the flow sequence start indicator? + if buf[pos] == '[' { + return yaml_parser_fetch_flow_collection_start(parser, yaml_FLOW_SEQUENCE_START_TOKEN) + } + + // Is it the flow mapping start indicator? + if parser.buffer[parser.buffer_pos] == '{' { + return yaml_parser_fetch_flow_collection_start(parser, yaml_FLOW_MAPPING_START_TOKEN) + } + + // Is it the flow sequence end indicator? + if parser.buffer[parser.buffer_pos] == ']' { + return yaml_parser_fetch_flow_collection_end(parser, + yaml_FLOW_SEQUENCE_END_TOKEN) + } + + // Is it the flow mapping end indicator? + if parser.buffer[parser.buffer_pos] == '}' { + return yaml_parser_fetch_flow_collection_end(parser, + yaml_FLOW_MAPPING_END_TOKEN) + } + + // Is it the flow entry indicator? + if parser.buffer[parser.buffer_pos] == ',' { + return yaml_parser_fetch_flow_entry(parser) + } + + // Is it the block entry indicator? + if parser.buffer[parser.buffer_pos] == '-' && is_blankz(parser.buffer, parser.buffer_pos+1) { + return yaml_parser_fetch_block_entry(parser) + } + + // Is it the key indicator? + if parser.buffer[parser.buffer_pos] == '?' && (parser.flow_level > 0 || is_blankz(parser.buffer, parser.buffer_pos+1)) { + return yaml_parser_fetch_key(parser) + } + + // Is it the value indicator? + if parser.buffer[parser.buffer_pos] == ':' && (parser.flow_level > 0 || is_blankz(parser.buffer, parser.buffer_pos+1)) { + return yaml_parser_fetch_value(parser) + } + + // Is it an alias? + if parser.buffer[parser.buffer_pos] == '*' { + return yaml_parser_fetch_anchor(parser, yaml_ALIAS_TOKEN) + } + + // Is it an anchor? + if parser.buffer[parser.buffer_pos] == '&' { + return yaml_parser_fetch_anchor(parser, yaml_ANCHOR_TOKEN) + } + + // Is it a tag? + if parser.buffer[parser.buffer_pos] == '!' { + return yaml_parser_fetch_tag(parser) + } + + // Is it a literal scalar? + if parser.buffer[parser.buffer_pos] == '|' && parser.flow_level == 0 { + return yaml_parser_fetch_block_scalar(parser, true) + } + + // Is it a folded scalar? + if parser.buffer[parser.buffer_pos] == '>' && parser.flow_level == 0 { + return yaml_parser_fetch_block_scalar(parser, false) + } + + // Is it a single-quoted scalar? + if parser.buffer[parser.buffer_pos] == '\'' { + return yaml_parser_fetch_flow_scalar(parser, true) + } + + // Is it a double-quoted scalar? + if parser.buffer[parser.buffer_pos] == '"' { + return yaml_parser_fetch_flow_scalar(parser, false) + } + + // Is it a plain scalar? + // + // A plain scalar may start with any non-blank characters except + // + // '-', '?', ':', ',', '[', ']', '{', '}', + // '#', '&', '*', '!', '|', '>', '\'', '\"', + // '%', '@', '`'. + // + // In the block context (and, for the '-' indicator, in the flow context + // too), it may also start with the characters + // + // '-', '?', ':' + // + // if it is followed by a non-space character. + // + // The last rule is more restrictive than the specification requires. + // [Go] Make this logic more reasonable. + //switch parser.buffer[parser.buffer_pos] { + //case '-', '?', ':', ',', '?', '-', ',', ':', ']', '[', '}', '{', '&', '#', '!', '*', '>', '|', '"', '\'', '@', '%', '-', '`': + //} + if !(is_blankz(parser.buffer, parser.buffer_pos) || parser.buffer[parser.buffer_pos] == '-' || + parser.buffer[parser.buffer_pos] == '?' || parser.buffer[parser.buffer_pos] == ':' || + parser.buffer[parser.buffer_pos] == ',' || parser.buffer[parser.buffer_pos] == '[' || + parser.buffer[parser.buffer_pos] == ']' || parser.buffer[parser.buffer_pos] == '{' || + parser.buffer[parser.buffer_pos] == '}' || parser.buffer[parser.buffer_pos] == '#' || + parser.buffer[parser.buffer_pos] == '&' || parser.buffer[parser.buffer_pos] == '*' || + parser.buffer[parser.buffer_pos] == '!' || parser.buffer[parser.buffer_pos] == '|' || + parser.buffer[parser.buffer_pos] == '>' || parser.buffer[parser.buffer_pos] == '\'' || + parser.buffer[parser.buffer_pos] == '"' || parser.buffer[parser.buffer_pos] == '%' || + parser.buffer[parser.buffer_pos] == '@' || parser.buffer[parser.buffer_pos] == '`') || + (parser.buffer[parser.buffer_pos] == '-' && !is_blank(parser.buffer, parser.buffer_pos+1)) || + (parser.flow_level == 0 && + (parser.buffer[parser.buffer_pos] == '?' || parser.buffer[parser.buffer_pos] == ':') && + !is_blankz(parser.buffer, parser.buffer_pos+1)) { + return yaml_parser_fetch_plain_scalar(parser) + } + + // If we don't determine the token type so far, it is an error. + return yaml_parser_set_scanner_error(parser, + "while scanning for the next token", parser.mark, + "found character that cannot start any token") +} + +// Check the list of potential simple keys and remove the positions that +// cannot contain simple keys anymore. +func yaml_parser_stale_simple_keys(parser *yaml_parser_t) bool { + // Check for a potential simple key for each flow level. + for i := range parser.simple_keys { + simple_key := &parser.simple_keys[i] + + // The specification requires that a simple key + // + // - is limited to a single line, + // - is shorter than 1024 characters. + if simple_key.possible && (simple_key.mark.line < parser.mark.line || simple_key.mark.index+1024 < parser.mark.index) { + + // Check if the potential simple key to be removed is required. + if simple_key.required { + return yaml_parser_set_scanner_error(parser, + "while scanning a simple key", simple_key.mark, + "could not find expected ':'") + } + simple_key.possible = false + } + } + return true +} + +// Check if a simple key may start at the current position and add it if +// needed. +func yaml_parser_save_simple_key(parser *yaml_parser_t) bool { + // A simple key is required at the current position if the scanner is in + // the block context and the current column coincides with the indentation + // level. + + required := parser.flow_level == 0 && parser.indent == parser.mark.column + + // + // If the current position may start a simple key, save it. + // + if parser.simple_key_allowed { + simple_key := yaml_simple_key_t{ + possible: true, + required: required, + token_number: parser.tokens_parsed + (len(parser.tokens) - parser.tokens_head), + } + simple_key.mark = parser.mark + + if !yaml_parser_remove_simple_key(parser) { + return false + } + parser.simple_keys[len(parser.simple_keys)-1] = simple_key + } + return true +} + +// Remove a potential simple key at the current flow level. +func yaml_parser_remove_simple_key(parser *yaml_parser_t) bool { + i := len(parser.simple_keys) - 1 + if parser.simple_keys[i].possible { + // If the key is required, it is an error. + if parser.simple_keys[i].required { + return yaml_parser_set_scanner_error(parser, + "while scanning a simple key", parser.simple_keys[i].mark, + "could not find expected ':'") + } + } + // Remove the key from the stack. + parser.simple_keys[i].possible = false + return true +} + +// Increase the flow level and resize the simple key list if needed. +func yaml_parser_increase_flow_level(parser *yaml_parser_t) bool { + // Reset the simple key on the next level. + parser.simple_keys = append(parser.simple_keys, yaml_simple_key_t{}) + + // Increase the flow level. + parser.flow_level++ + return true +} + +// Decrease the flow level. +func yaml_parser_decrease_flow_level(parser *yaml_parser_t) bool { + if parser.flow_level > 0 { + parser.flow_level-- + parser.simple_keys = parser.simple_keys[:len(parser.simple_keys)-1] + } + return true +} + +// Push the current indentation level to the stack and set the new level +// the current column is greater than the indentation level. In this case, +// append or insert the specified token into the token queue. +func yaml_parser_roll_indent(parser *yaml_parser_t, column, number int, typ yaml_token_type_t, mark yaml_mark_t) bool { + // In the flow context, do nothing. + if parser.flow_level > 0 { + return true + } + + if parser.indent < column { + // Push the current indentation level to the stack and set the new + // indentation level. + parser.indents = append(parser.indents, parser.indent) + parser.indent = column + + // Create a token and insert it into the queue. + token := yaml_token_t{ + typ: typ, + start_mark: mark, + end_mark: mark, + } + if number > -1 { + number -= parser.tokens_parsed + } + yaml_insert_token(parser, number, &token) + } + return true +} + +// Pop indentation levels from the indents stack until the current level +// becomes less or equal to the column. For each indentation level, append +// the BLOCK-END token. +func yaml_parser_unroll_indent(parser *yaml_parser_t, column int) bool { + // In the flow context, do nothing. + if parser.flow_level > 0 { + return true + } + + // Loop through the indentation levels in the stack. + for parser.indent > column { + // Create a token and append it to the queue. + token := yaml_token_t{ + typ: yaml_BLOCK_END_TOKEN, + start_mark: parser.mark, + end_mark: parser.mark, + } + yaml_insert_token(parser, -1, &token) + + // Pop the indentation level. + parser.indent = parser.indents[len(parser.indents)-1] + parser.indents = parser.indents[:len(parser.indents)-1] + } + return true +} + +// Initialize the scanner and produce the STREAM-START token. +func yaml_parser_fetch_stream_start(parser *yaml_parser_t) bool { + + // Set the initial indentation. + parser.indent = -1 + + // Initialize the simple key stack. + parser.simple_keys = append(parser.simple_keys, yaml_simple_key_t{}) + + // A simple key is allowed at the beginning of the stream. + parser.simple_key_allowed = true + + // We have started. + parser.stream_start_produced = true + + // Create the STREAM-START token and append it to the queue. + token := yaml_token_t{ + typ: yaml_STREAM_START_TOKEN, + start_mark: parser.mark, + end_mark: parser.mark, + encoding: parser.encoding, + } + yaml_insert_token(parser, -1, &token) + return true +} + +// Produce the STREAM-END token and shut down the scanner. +func yaml_parser_fetch_stream_end(parser *yaml_parser_t) bool { + + // Force new line. + if parser.mark.column != 0 { + parser.mark.column = 0 + parser.mark.line++ + } + + // Reset the indentation level. + if !yaml_parser_unroll_indent(parser, -1) { + return false + } + + // Reset simple keys. + if !yaml_parser_remove_simple_key(parser) { + return false + } + + parser.simple_key_allowed = false + + // Create the STREAM-END token and append it to the queue. + token := yaml_token_t{ + typ: yaml_STREAM_END_TOKEN, + start_mark: parser.mark, + end_mark: parser.mark, + } + yaml_insert_token(parser, -1, &token) + return true +} + +// Produce a VERSION-DIRECTIVE or TAG-DIRECTIVE token. +func yaml_parser_fetch_directive(parser *yaml_parser_t) bool { + // Reset the indentation level. + if !yaml_parser_unroll_indent(parser, -1) { + return false + } + + // Reset simple keys. + if !yaml_parser_remove_simple_key(parser) { + return false + } + + parser.simple_key_allowed = false + + // Create the YAML-DIRECTIVE or TAG-DIRECTIVE token. + token := yaml_token_t{} + if !yaml_parser_scan_directive(parser, &token) { + return false + } + // Append the token to the queue. + yaml_insert_token(parser, -1, &token) + return true +} + +// Produce the DOCUMENT-START or DOCUMENT-END token. +func yaml_parser_fetch_document_indicator(parser *yaml_parser_t, typ yaml_token_type_t) bool { + // Reset the indentation level. + if !yaml_parser_unroll_indent(parser, -1) { + return false + } + + // Reset simple keys. + if !yaml_parser_remove_simple_key(parser) { + return false + } + + parser.simple_key_allowed = false + + // Consume the token. + start_mark := parser.mark + + skip(parser) + skip(parser) + skip(parser) + + end_mark := parser.mark + + // Create the DOCUMENT-START or DOCUMENT-END token. + token := yaml_token_t{ + typ: typ, + start_mark: start_mark, + end_mark: end_mark, + } + // Append the token to the queue. + yaml_insert_token(parser, -1, &token) + return true +} + +// Produce the FLOW-SEQUENCE-START or FLOW-MAPPING-START token. +func yaml_parser_fetch_flow_collection_start(parser *yaml_parser_t, typ yaml_token_type_t) bool { + // The indicators '[' and '{' may start a simple key. + if !yaml_parser_save_simple_key(parser) { + return false + } + + // Increase the flow level. + if !yaml_parser_increase_flow_level(parser) { + return false + } + + // A simple key may follow the indicators '[' and '{'. + parser.simple_key_allowed = true + + // Consume the token. + start_mark := parser.mark + skip(parser) + end_mark := parser.mark + + // Create the FLOW-SEQUENCE-START of FLOW-MAPPING-START token. + token := yaml_token_t{ + typ: typ, + start_mark: start_mark, + end_mark: end_mark, + } + // Append the token to the queue. + yaml_insert_token(parser, -1, &token) + return true +} + +// Produce the FLOW-SEQUENCE-END or FLOW-MAPPING-END token. +func yaml_parser_fetch_flow_collection_end(parser *yaml_parser_t, typ yaml_token_type_t) bool { + // Reset any potential simple key on the current flow level. + if !yaml_parser_remove_simple_key(parser) { + return false + } + + // Decrease the flow level. + if !yaml_parser_decrease_flow_level(parser) { + return false + } + + // No simple keys after the indicators ']' and '}'. + parser.simple_key_allowed = false + + // Consume the token. + + start_mark := parser.mark + skip(parser) + end_mark := parser.mark + + // Create the FLOW-SEQUENCE-END of FLOW-MAPPING-END token. + token := yaml_token_t{ + typ: typ, + start_mark: start_mark, + end_mark: end_mark, + } + // Append the token to the queue. + yaml_insert_token(parser, -1, &token) + return true +} + +// Produce the FLOW-ENTRY token. +func yaml_parser_fetch_flow_entry(parser *yaml_parser_t) bool { + // Reset any potential simple keys on the current flow level. + if !yaml_parser_remove_simple_key(parser) { + return false + } + + // Simple keys are allowed after ','. + parser.simple_key_allowed = true + + // Consume the token. + start_mark := parser.mark + skip(parser) + end_mark := parser.mark + + // Create the FLOW-ENTRY token and append it to the queue. + token := yaml_token_t{ + typ: yaml_FLOW_ENTRY_TOKEN, + start_mark: start_mark, + end_mark: end_mark, + } + yaml_insert_token(parser, -1, &token) + return true +} + +// Produce the BLOCK-ENTRY token. +func yaml_parser_fetch_block_entry(parser *yaml_parser_t) bool { + // Check if the scanner is in the block context. + if parser.flow_level == 0 { + // Check if we are allowed to start a new entry. + if !parser.simple_key_allowed { + return yaml_parser_set_scanner_error(parser, "", parser.mark, + "block sequence entries are not allowed in this context") + } + // Add the BLOCK-SEQUENCE-START token if needed. + if !yaml_parser_roll_indent(parser, parser.mark.column, -1, yaml_BLOCK_SEQUENCE_START_TOKEN, parser.mark) { + return false + } + } else { + // It is an error for the '-' indicator to occur in the flow context, + // but we let the Parser detect and report about it because the Parser + // is able to point to the context. + } + + // Reset any potential simple keys on the current flow level. + if !yaml_parser_remove_simple_key(parser) { + return false + } + + // Simple keys are allowed after '-'. + parser.simple_key_allowed = true + + // Consume the token. + start_mark := parser.mark + skip(parser) + end_mark := parser.mark + + // Create the BLOCK-ENTRY token and append it to the queue. + token := yaml_token_t{ + typ: yaml_BLOCK_ENTRY_TOKEN, + start_mark: start_mark, + end_mark: end_mark, + } + yaml_insert_token(parser, -1, &token) + return true +} + +// Produce the KEY token. +func yaml_parser_fetch_key(parser *yaml_parser_t) bool { + + // In the block context, additional checks are required. + if parser.flow_level == 0 { + // Check if we are allowed to start a new key (not nessesary simple). + if !parser.simple_key_allowed { + return yaml_parser_set_scanner_error(parser, "", parser.mark, + "mapping keys are not allowed in this context") + } + // Add the BLOCK-MAPPING-START token if needed. + if !yaml_parser_roll_indent(parser, parser.mark.column, -1, yaml_BLOCK_MAPPING_START_TOKEN, parser.mark) { + return false + } + } + + // Reset any potential simple keys on the current flow level. + if !yaml_parser_remove_simple_key(parser) { + return false + } + + // Simple keys are allowed after '?' in the block context. + parser.simple_key_allowed = parser.flow_level == 0 + + // Consume the token. + start_mark := parser.mark + skip(parser) + end_mark := parser.mark + + // Create the KEY token and append it to the queue. + token := yaml_token_t{ + typ: yaml_KEY_TOKEN, + start_mark: start_mark, + end_mark: end_mark, + } + yaml_insert_token(parser, -1, &token) + return true +} + +// Produce the VALUE token. +func yaml_parser_fetch_value(parser *yaml_parser_t) bool { + + simple_key := &parser.simple_keys[len(parser.simple_keys)-1] + + // Have we found a simple key? + if simple_key.possible { + // Create the KEY token and insert it into the queue. + token := yaml_token_t{ + typ: yaml_KEY_TOKEN, + start_mark: simple_key.mark, + end_mark: simple_key.mark, + } + yaml_insert_token(parser, simple_key.token_number-parser.tokens_parsed, &token) + + // In the block context, we may need to add the BLOCK-MAPPING-START token. + if !yaml_parser_roll_indent(parser, simple_key.mark.column, + simple_key.token_number, + yaml_BLOCK_MAPPING_START_TOKEN, simple_key.mark) { + return false + } + + // Remove the simple key. + simple_key.possible = false + + // A simple key cannot follow another simple key. + parser.simple_key_allowed = false + + } else { + // The ':' indicator follows a complex key. + + // In the block context, extra checks are required. + if parser.flow_level == 0 { + + // Check if we are allowed to start a complex value. + if !parser.simple_key_allowed { + return yaml_parser_set_scanner_error(parser, "", parser.mark, + "mapping values are not allowed in this context") + } + + // Add the BLOCK-MAPPING-START token if needed. + if !yaml_parser_roll_indent(parser, parser.mark.column, -1, yaml_BLOCK_MAPPING_START_TOKEN, parser.mark) { + return false + } + } + + // Simple keys after ':' are allowed in the block context. + parser.simple_key_allowed = parser.flow_level == 0 + } + + // Consume the token. + start_mark := parser.mark + skip(parser) + end_mark := parser.mark + + // Create the VALUE token and append it to the queue. + token := yaml_token_t{ + typ: yaml_VALUE_TOKEN, + start_mark: start_mark, + end_mark: end_mark, + } + yaml_insert_token(parser, -1, &token) + return true +} + +// Produce the ALIAS or ANCHOR token. +func yaml_parser_fetch_anchor(parser *yaml_parser_t, typ yaml_token_type_t) bool { + // An anchor or an alias could be a simple key. + if !yaml_parser_save_simple_key(parser) { + return false + } + + // A simple key cannot follow an anchor or an alias. + parser.simple_key_allowed = false + + // Create the ALIAS or ANCHOR token and append it to the queue. + var token yaml_token_t + if !yaml_parser_scan_anchor(parser, &token, typ) { + return false + } + yaml_insert_token(parser, -1, &token) + return true +} + +// Produce the TAG token. +func yaml_parser_fetch_tag(parser *yaml_parser_t) bool { + // A tag could be a simple key. + if !yaml_parser_save_simple_key(parser) { + return false + } + + // A simple key cannot follow a tag. + parser.simple_key_allowed = false + + // Create the TAG token and append it to the queue. + var token yaml_token_t + if !yaml_parser_scan_tag(parser, &token) { + return false + } + yaml_insert_token(parser, -1, &token) + return true +} + +// Produce the SCALAR(...,literal) or SCALAR(...,folded) tokens. +func yaml_parser_fetch_block_scalar(parser *yaml_parser_t, literal bool) bool { + // Remove any potential simple keys. + if !yaml_parser_remove_simple_key(parser) { + return false + } + + // A simple key may follow a block scalar. + parser.simple_key_allowed = true + + // Create the SCALAR token and append it to the queue. + var token yaml_token_t + if !yaml_parser_scan_block_scalar(parser, &token, literal) { + return false + } + yaml_insert_token(parser, -1, &token) + return true +} + +// Produce the SCALAR(...,single-quoted) or SCALAR(...,double-quoted) tokens. +func yaml_parser_fetch_flow_scalar(parser *yaml_parser_t, single bool) bool { + // A plain scalar could be a simple key. + if !yaml_parser_save_simple_key(parser) { + return false + } + + // A simple key cannot follow a flow scalar. + parser.simple_key_allowed = false + + // Create the SCALAR token and append it to the queue. + var token yaml_token_t + if !yaml_parser_scan_flow_scalar(parser, &token, single) { + return false + } + yaml_insert_token(parser, -1, &token) + return true +} + +// Produce the SCALAR(...,plain) token. +func yaml_parser_fetch_plain_scalar(parser *yaml_parser_t) bool { + // A plain scalar could be a simple key. + if !yaml_parser_save_simple_key(parser) { + return false + } + + // A simple key cannot follow a flow scalar. + parser.simple_key_allowed = false + + // Create the SCALAR token and append it to the queue. + var token yaml_token_t + if !yaml_parser_scan_plain_scalar(parser, &token) { + return false + } + yaml_insert_token(parser, -1, &token) + return true +} + +// Eat whitespaces and comments until the next token is found. +func yaml_parser_scan_to_next_token(parser *yaml_parser_t) bool { + + // Until the next token is not found. + for { + // Allow the BOM mark to start a line. + if parser.unread < 1 && !yaml_parser_update_buffer(parser, 1) { + return false + } + if parser.mark.column == 0 && is_bom(parser.buffer, parser.buffer_pos) { + skip(parser) + } + + // Eat whitespaces. + // Tabs are allowed: + // - in the flow context + // - in the block context, but not at the beginning of the line or + // after '-', '?', or ':' (complex value). + if parser.unread < 1 && !yaml_parser_update_buffer(parser, 1) { + return false + } + + for parser.buffer[parser.buffer_pos] == ' ' || ((parser.flow_level > 0 || !parser.simple_key_allowed) && parser.buffer[parser.buffer_pos] == '\t') { + skip(parser) + if parser.unread < 1 && !yaml_parser_update_buffer(parser, 1) { + return false + } + } + + // Eat a comment until a line break. + if parser.buffer[parser.buffer_pos] == '#' { + for !is_breakz(parser.buffer, parser.buffer_pos) { + skip(parser) + if parser.unread < 1 && !yaml_parser_update_buffer(parser, 1) { + return false + } + } + } + + // If it is a line break, eat it. + if is_break(parser.buffer, parser.buffer_pos) { + if parser.unread < 2 && !yaml_parser_update_buffer(parser, 2) { + return false + } + skip_line(parser) + + // In the block context, a new line may start a simple key. + if parser.flow_level == 0 { + parser.simple_key_allowed = true + } + } else { + break // We have found a token. + } + } + + return true +} + +// Scan a YAML-DIRECTIVE or TAG-DIRECTIVE token. +// +// Scope: +// %YAML 1.1 # a comment \n +// ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +// %TAG !yaml! tag:yaml.org,2002: \n +// ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +// +func yaml_parser_scan_directive(parser *yaml_parser_t, token *yaml_token_t) bool { + // Eat '%'. + start_mark := parser.mark + skip(parser) + + // Scan the directive name. + var name []byte + if !yaml_parser_scan_directive_name(parser, start_mark, &name) { + return false + } + + // Is it a YAML directive? + if bytes.Equal(name, []byte("YAML")) { + // Scan the VERSION directive value. + var major, minor int8 + if !yaml_parser_scan_version_directive_value(parser, start_mark, &major, &minor) { + return false + } + end_mark := parser.mark + + // Create a VERSION-DIRECTIVE token. + *token = yaml_token_t{ + typ: yaml_VERSION_DIRECTIVE_TOKEN, + start_mark: start_mark, + end_mark: end_mark, + major: major, + minor: minor, + } + + // Is it a TAG directive? + } else if bytes.Equal(name, []byte("TAG")) { + // Scan the TAG directive value. + var handle, prefix []byte + if !yaml_parser_scan_tag_directive_value(parser, start_mark, &handle, &prefix) { + return false + } + end_mark := parser.mark + + // Create a TAG-DIRECTIVE token. + *token = yaml_token_t{ + typ: yaml_TAG_DIRECTIVE_TOKEN, + start_mark: start_mark, + end_mark: end_mark, + value: handle, + prefix: prefix, + } + + // Unknown directive. + } else { + yaml_parser_set_scanner_error(parser, "while scanning a directive", + start_mark, "found unknown directive name") + return false + } + + // Eat the rest of the line including any comments. + if parser.unread < 1 && !yaml_parser_update_buffer(parser, 1) { + return false + } + + for is_blank(parser.buffer, parser.buffer_pos) { + skip(parser) + if parser.unread < 1 && !yaml_parser_update_buffer(parser, 1) { + return false + } + } + + if parser.buffer[parser.buffer_pos] == '#' { + for !is_breakz(parser.buffer, parser.buffer_pos) { + skip(parser) + if parser.unread < 1 && !yaml_parser_update_buffer(parser, 1) { + return false + } + } + } + + // Check if we are at the end of the line. + if !is_breakz(parser.buffer, parser.buffer_pos) { + yaml_parser_set_scanner_error(parser, "while scanning a directive", + start_mark, "did not find expected comment or line break") + return false + } + + // Eat a line break. + if is_break(parser.buffer, parser.buffer_pos) { + if parser.unread < 2 && !yaml_parser_update_buffer(parser, 2) { + return false + } + skip_line(parser) + } + + return true +} + +// Scan the directive name. +// +// Scope: +// %YAML 1.1 # a comment \n +// ^^^^ +// %TAG !yaml! tag:yaml.org,2002: \n +// ^^^ +// +func yaml_parser_scan_directive_name(parser *yaml_parser_t, start_mark yaml_mark_t, name *[]byte) bool { + // Consume the directive name. + if parser.unread < 1 && !yaml_parser_update_buffer(parser, 1) { + return false + } + + var s []byte + for is_alpha(parser.buffer, parser.buffer_pos) { + s = read(parser, s) + if parser.unread < 1 && !yaml_parser_update_buffer(parser, 1) { + return false + } + } + + // Check if the name is empty. + if len(s) == 0 { + yaml_parser_set_scanner_error(parser, "while scanning a directive", + start_mark, "could not find expected directive name") + return false + } + + // Check for an blank character after the name. + if !is_blankz(parser.buffer, parser.buffer_pos) { + yaml_parser_set_scanner_error(parser, "while scanning a directive", + start_mark, "found unexpected non-alphabetical character") + return false + } + *name = s + return true +} + +// Scan the value of VERSION-DIRECTIVE. +// +// Scope: +// %YAML 1.1 # a comment \n +// ^^^^^^ +func yaml_parser_scan_version_directive_value(parser *yaml_parser_t, start_mark yaml_mark_t, major, minor *int8) bool { + // Eat whitespaces. + if parser.unread < 1 && !yaml_parser_update_buffer(parser, 1) { + return false + } + for is_blank(parser.buffer, parser.buffer_pos) { + skip(parser) + if parser.unread < 1 && !yaml_parser_update_buffer(parser, 1) { + return false + } + } + + // Consume the major version number. + if !yaml_parser_scan_version_directive_number(parser, start_mark, major) { + return false + } + + // Eat '.'. + if parser.buffer[parser.buffer_pos] != '.' { + return yaml_parser_set_scanner_error(parser, "while scanning a %YAML directive", + start_mark, "did not find expected digit or '.' character") + } + + skip(parser) + + // Consume the minor version number. + if !yaml_parser_scan_version_directive_number(parser, start_mark, minor) { + return false + } + return true +} + +const max_number_length = 2 + +// Scan the version number of VERSION-DIRECTIVE. +// +// Scope: +// %YAML 1.1 # a comment \n +// ^ +// %YAML 1.1 # a comment \n +// ^ +func yaml_parser_scan_version_directive_number(parser *yaml_parser_t, start_mark yaml_mark_t, number *int8) bool { + + // Repeat while the next character is digit. + if parser.unread < 1 && !yaml_parser_update_buffer(parser, 1) { + return false + } + var value, length int8 + for is_digit(parser.buffer, parser.buffer_pos) { + // Check if the number is too long. + length++ + if length > max_number_length { + return yaml_parser_set_scanner_error(parser, "while scanning a %YAML directive", + start_mark, "found extremely long version number") + } + value = value*10 + int8(as_digit(parser.buffer, parser.buffer_pos)) + skip(parser) + if parser.unread < 1 && !yaml_parser_update_buffer(parser, 1) { + return false + } + } + + // Check if the number was present. + if length == 0 { + return yaml_parser_set_scanner_error(parser, "while scanning a %YAML directive", + start_mark, "did not find expected version number") + } + *number = value + return true +} + +// Scan the value of a TAG-DIRECTIVE token. +// +// Scope: +// %TAG !yaml! tag:yaml.org,2002: \n +// ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +// +func yaml_parser_scan_tag_directive_value(parser *yaml_parser_t, start_mark yaml_mark_t, handle, prefix *[]byte) bool { + var handle_value, prefix_value []byte + + // Eat whitespaces. + if parser.unread < 1 && !yaml_parser_update_buffer(parser, 1) { + return false + } + + for is_blank(parser.buffer, parser.buffer_pos) { + skip(parser) + if parser.unread < 1 && !yaml_parser_update_buffer(parser, 1) { + return false + } + } + + // Scan a handle. + if !yaml_parser_scan_tag_handle(parser, true, start_mark, &handle_value) { + return false + } + + // Expect a whitespace. + if parser.unread < 1 && !yaml_parser_update_buffer(parser, 1) { + return false + } + if !is_blank(parser.buffer, parser.buffer_pos) { + yaml_parser_set_scanner_error(parser, "while scanning a %TAG directive", + start_mark, "did not find expected whitespace") + return false + } + + // Eat whitespaces. + for is_blank(parser.buffer, parser.buffer_pos) { + skip(parser) + if parser.unread < 1 && !yaml_parser_update_buffer(parser, 1) { + return false + } + } + + // Scan a prefix. + if !yaml_parser_scan_tag_uri(parser, true, nil, start_mark, &prefix_value) { + return false + } + + // Expect a whitespace or line break. + if parser.unread < 1 && !yaml_parser_update_buffer(parser, 1) { + return false + } + if !is_blankz(parser.buffer, parser.buffer_pos) { + yaml_parser_set_scanner_error(parser, "while scanning a %TAG directive", + start_mark, "did not find expected whitespace or line break") + return false + } + + *handle = handle_value + *prefix = prefix_value + return true +} + +func yaml_parser_scan_anchor(parser *yaml_parser_t, token *yaml_token_t, typ yaml_token_type_t) bool { + var s []byte + + // Eat the indicator character. + start_mark := parser.mark + skip(parser) + + // Consume the value. + if parser.unread < 1 && !yaml_parser_update_buffer(parser, 1) { + return false + } + + for is_alpha(parser.buffer, parser.buffer_pos) { + s = read(parser, s) + if parser.unread < 1 && !yaml_parser_update_buffer(parser, 1) { + return false + } + } + + end_mark := parser.mark + + /* + * Check if length of the anchor is greater than 0 and it is followed by + * a whitespace character or one of the indicators: + * + * '?', ':', ',', ']', '}', '%', '@', '`'. + */ + + if len(s) == 0 || + !(is_blankz(parser.buffer, parser.buffer_pos) || parser.buffer[parser.buffer_pos] == '?' || + parser.buffer[parser.buffer_pos] == ':' || parser.buffer[parser.buffer_pos] == ',' || + parser.buffer[parser.buffer_pos] == ']' || parser.buffer[parser.buffer_pos] == '}' || + parser.buffer[parser.buffer_pos] == '%' || parser.buffer[parser.buffer_pos] == '@' || + parser.buffer[parser.buffer_pos] == '`') { + context := "while scanning an alias" + if typ == yaml_ANCHOR_TOKEN { + context = "while scanning an anchor" + } + yaml_parser_set_scanner_error(parser, context, start_mark, + "did not find expected alphabetic or numeric character") + return false + } + + // Create a token. + *token = yaml_token_t{ + typ: typ, + start_mark: start_mark, + end_mark: end_mark, + value: s, + } + + return true +} + +/* + * Scan a TAG token. + */ + +func yaml_parser_scan_tag(parser *yaml_parser_t, token *yaml_token_t) bool { + var handle, suffix []byte + + start_mark := parser.mark + + // Check if the tag is in the canonical form. + if parser.unread < 2 && !yaml_parser_update_buffer(parser, 2) { + return false + } + + if parser.buffer[parser.buffer_pos+1] == '<' { + // Keep the handle as '' + + // Eat '!<' + skip(parser) + skip(parser) + + // Consume the tag value. + if !yaml_parser_scan_tag_uri(parser, false, nil, start_mark, &suffix) { + return false + } + + // Check for '>' and eat it. + if parser.buffer[parser.buffer_pos] != '>' { + yaml_parser_set_scanner_error(parser, "while scanning a tag", + start_mark, "did not find the expected '>'") + return false + } + + skip(parser) + } else { + // The tag has either the '!suffix' or the '!handle!suffix' form. + + // First, try to scan a handle. + if !yaml_parser_scan_tag_handle(parser, false, start_mark, &handle) { + return false + } + + // Check if it is, indeed, handle. + if handle[0] == '!' && len(handle) > 1 && handle[len(handle)-1] == '!' { + // Scan the suffix now. + if !yaml_parser_scan_tag_uri(parser, false, nil, start_mark, &suffix) { + return false + } + } else { + // It wasn't a handle after all. Scan the rest of the tag. + if !yaml_parser_scan_tag_uri(parser, false, handle, start_mark, &suffix) { + return false + } + + // Set the handle to '!'. + handle = []byte{'!'} + + // A special case: the '!' tag. Set the handle to '' and the + // suffix to '!'. + if len(suffix) == 0 { + handle, suffix = suffix, handle + } + } + } + + // Check the character which ends the tag. + if parser.unread < 1 && !yaml_parser_update_buffer(parser, 1) { + return false + } + if !is_blankz(parser.buffer, parser.buffer_pos) { + yaml_parser_set_scanner_error(parser, "while scanning a tag", + start_mark, "did not find expected whitespace or line break") + return false + } + + end_mark := parser.mark + + // Create a token. + *token = yaml_token_t{ + typ: yaml_TAG_TOKEN, + start_mark: start_mark, + end_mark: end_mark, + value: handle, + suffix: suffix, + } + return true +} + +// Scan a tag handle. +func yaml_parser_scan_tag_handle(parser *yaml_parser_t, directive bool, start_mark yaml_mark_t, handle *[]byte) bool { + // Check the initial '!' character. + if parser.unread < 1 && !yaml_parser_update_buffer(parser, 1) { + return false + } + if parser.buffer[parser.buffer_pos] != '!' { + yaml_parser_set_scanner_tag_error(parser, directive, + start_mark, "did not find expected '!'") + return false + } + + var s []byte + + // Copy the '!' character. + s = read(parser, s) + + // Copy all subsequent alphabetical and numerical characters. + if parser.unread < 1 && !yaml_parser_update_buffer(parser, 1) { + return false + } + for is_alpha(parser.buffer, parser.buffer_pos) { + s = read(parser, s) + if parser.unread < 1 && !yaml_parser_update_buffer(parser, 1) { + return false + } + } + + // Check if the trailing character is '!' and copy it. + if parser.buffer[parser.buffer_pos] == '!' { + s = read(parser, s) + } else { + // It's either the '!' tag or not really a tag handle. If it's a %TAG + // directive, it's an error. If it's a tag token, it must be a part of URI. + if directive && string(s) != "!" { + yaml_parser_set_scanner_tag_error(parser, directive, + start_mark, "did not find expected '!'") + return false + } + } + + *handle = s + return true +} + +// Scan a tag. +func yaml_parser_scan_tag_uri(parser *yaml_parser_t, directive bool, head []byte, start_mark yaml_mark_t, uri *[]byte) bool { + //size_t length = head ? strlen((char *)head) : 0 + var s []byte + hasTag := len(head) > 0 + + // Copy the head if needed. + // + // Note that we don't copy the leading '!' character. + if len(head) > 1 { + s = append(s, head[1:]...) + } + + // Scan the tag. + if parser.unread < 1 && !yaml_parser_update_buffer(parser, 1) { + return false + } + + // The set of characters that may appear in URI is as follows: + // + // '0'-'9', 'A'-'Z', 'a'-'z', '_', '-', ';', '/', '?', ':', '@', '&', + // '=', '+', '$', ',', '.', '!', '~', '*', '\'', '(', ')', '[', ']', + // '%'. + // [Go] Convert this into more reasonable logic. + for is_alpha(parser.buffer, parser.buffer_pos) || parser.buffer[parser.buffer_pos] == ';' || + parser.buffer[parser.buffer_pos] == '/' || parser.buffer[parser.buffer_pos] == '?' || + parser.buffer[parser.buffer_pos] == ':' || parser.buffer[parser.buffer_pos] == '@' || + parser.buffer[parser.buffer_pos] == '&' || parser.buffer[parser.buffer_pos] == '=' || + parser.buffer[parser.buffer_pos] == '+' || parser.buffer[parser.buffer_pos] == '$' || + parser.buffer[parser.buffer_pos] == ',' || parser.buffer[parser.buffer_pos] == '.' || + parser.buffer[parser.buffer_pos] == '!' || parser.buffer[parser.buffer_pos] == '~' || + parser.buffer[parser.buffer_pos] == '*' || parser.buffer[parser.buffer_pos] == '\'' || + parser.buffer[parser.buffer_pos] == '(' || parser.buffer[parser.buffer_pos] == ')' || + parser.buffer[parser.buffer_pos] == '[' || parser.buffer[parser.buffer_pos] == ']' || + parser.buffer[parser.buffer_pos] == '%' { + // Check if it is a URI-escape sequence. + if parser.buffer[parser.buffer_pos] == '%' { + if !yaml_parser_scan_uri_escapes(parser, directive, start_mark, &s) { + return false + } + } else { + s = read(parser, s) + } + if parser.unread < 1 && !yaml_parser_update_buffer(parser, 1) { + return false + } + hasTag = true + } + + if !hasTag { + yaml_parser_set_scanner_tag_error(parser, directive, + start_mark, "did not find expected tag URI") + return false + } + *uri = s + return true +} + +// Decode an URI-escape sequence corresponding to a single UTF-8 character. +func yaml_parser_scan_uri_escapes(parser *yaml_parser_t, directive bool, start_mark yaml_mark_t, s *[]byte) bool { + + // Decode the required number of characters. + w := 1024 + for w > 0 { + // Check for a URI-escaped octet. + if parser.unread < 3 && !yaml_parser_update_buffer(parser, 3) { + return false + } + + if !(parser.buffer[parser.buffer_pos] == '%' && + is_hex(parser.buffer, parser.buffer_pos+1) && + is_hex(parser.buffer, parser.buffer_pos+2)) { + return yaml_parser_set_scanner_tag_error(parser, directive, + start_mark, "did not find URI escaped octet") + } + + // Get the octet. + octet := byte((as_hex(parser.buffer, parser.buffer_pos+1) << 4) + as_hex(parser.buffer, parser.buffer_pos+2)) + + // If it is the leading octet, determine the length of the UTF-8 sequence. + if w == 1024 { + w = width(octet) + if w == 0 { + return yaml_parser_set_scanner_tag_error(parser, directive, + start_mark, "found an incorrect leading UTF-8 octet") + } + } else { + // Check if the trailing octet is correct. + if octet&0xC0 != 0x80 { + return yaml_parser_set_scanner_tag_error(parser, directive, + start_mark, "found an incorrect trailing UTF-8 octet") + } + } + + // Copy the octet and move the pointers. + *s = append(*s, octet) + skip(parser) + skip(parser) + skip(parser) + w-- + } + return true +} + +// Scan a block scalar. +func yaml_parser_scan_block_scalar(parser *yaml_parser_t, token *yaml_token_t, literal bool) bool { + // Eat the indicator '|' or '>'. + start_mark := parser.mark + skip(parser) + + // Scan the additional block scalar indicators. + if parser.unread < 1 && !yaml_parser_update_buffer(parser, 1) { + return false + } + + // Check for a chomping indicator. + var chomping, increment int + if parser.buffer[parser.buffer_pos] == '+' || parser.buffer[parser.buffer_pos] == '-' { + // Set the chomping method and eat the indicator. + if parser.buffer[parser.buffer_pos] == '+' { + chomping = +1 + } else { + chomping = -1 + } + skip(parser) + + // Check for an indentation indicator. + if parser.unread < 1 && !yaml_parser_update_buffer(parser, 1) { + return false + } + if is_digit(parser.buffer, parser.buffer_pos) { + // Check that the indentation is greater than 0. + if parser.buffer[parser.buffer_pos] == '0' { + yaml_parser_set_scanner_error(parser, "while scanning a block scalar", + start_mark, "found an indentation indicator equal to 0") + return false + } + + // Get the indentation level and eat the indicator. + increment = as_digit(parser.buffer, parser.buffer_pos) + skip(parser) + } + + } else if is_digit(parser.buffer, parser.buffer_pos) { + // Do the same as above, but in the opposite order. + + if parser.buffer[parser.buffer_pos] == '0' { + yaml_parser_set_scanner_error(parser, "while scanning a block scalar", + start_mark, "found an indentation indicator equal to 0") + return false + } + increment = as_digit(parser.buffer, parser.buffer_pos) + skip(parser) + + if parser.unread < 1 && !yaml_parser_update_buffer(parser, 1) { + return false + } + if parser.buffer[parser.buffer_pos] == '+' || parser.buffer[parser.buffer_pos] == '-' { + if parser.buffer[parser.buffer_pos] == '+' { + chomping = +1 + } else { + chomping = -1 + } + skip(parser) + } + } + + // Eat whitespaces and comments to the end of the line. + if parser.unread < 1 && !yaml_parser_update_buffer(parser, 1) { + return false + } + for is_blank(parser.buffer, parser.buffer_pos) { + skip(parser) + if parser.unread < 1 && !yaml_parser_update_buffer(parser, 1) { + return false + } + } + if parser.buffer[parser.buffer_pos] == '#' { + for !is_breakz(parser.buffer, parser.buffer_pos) { + skip(parser) + if parser.unread < 1 && !yaml_parser_update_buffer(parser, 1) { + return false + } + } + } + + // Check if we are at the end of the line. + if !is_breakz(parser.buffer, parser.buffer_pos) { + yaml_parser_set_scanner_error(parser, "while scanning a block scalar", + start_mark, "did not find expected comment or line break") + return false + } + + // Eat a line break. + if is_break(parser.buffer, parser.buffer_pos) { + if parser.unread < 2 && !yaml_parser_update_buffer(parser, 2) { + return false + } + skip_line(parser) + } + + end_mark := parser.mark + + // Set the indentation level if it was specified. + var indent int + if increment > 0 { + if parser.indent >= 0 { + indent = parser.indent + increment + } else { + indent = increment + } + } + + // Scan the leading line breaks and determine the indentation level if needed. + var s, leading_break, trailing_breaks []byte + if !yaml_parser_scan_block_scalar_breaks(parser, &indent, &trailing_breaks, start_mark, &end_mark) { + return false + } + + // Scan the block scalar content. + if parser.unread < 1 && !yaml_parser_update_buffer(parser, 1) { + return false + } + var leading_blank, trailing_blank bool + for parser.mark.column == indent && !is_z(parser.buffer, parser.buffer_pos) { + // We are at the beginning of a non-empty line. + + // Is it a trailing whitespace? + trailing_blank = is_blank(parser.buffer, parser.buffer_pos) + + // Check if we need to fold the leading line break. + if !literal && !leading_blank && !trailing_blank && len(leading_break) > 0 && leading_break[0] == '\n' { + // Do we need to join the lines by space? + if len(trailing_breaks) == 0 { + s = append(s, ' ') + } + } else { + s = append(s, leading_break...) + } + leading_break = leading_break[:0] + + // Append the remaining line breaks. + s = append(s, trailing_breaks...) + trailing_breaks = trailing_breaks[:0] + + // Is it a leading whitespace? + leading_blank = is_blank(parser.buffer, parser.buffer_pos) + + // Consume the current line. + for !is_breakz(parser.buffer, parser.buffer_pos) { + s = read(parser, s) + if parser.unread < 1 && !yaml_parser_update_buffer(parser, 1) { + return false + } + } + + // Consume the line break. + if parser.unread < 2 && !yaml_parser_update_buffer(parser, 2) { + return false + } + + leading_break = read_line(parser, leading_break) + + // Eat the following indentation spaces and line breaks. + if !yaml_parser_scan_block_scalar_breaks(parser, &indent, &trailing_breaks, start_mark, &end_mark) { + return false + } + } + + // Chomp the tail. + if chomping != -1 { + s = append(s, leading_break...) + } + if chomping == 1 { + s = append(s, trailing_breaks...) + } + + // Create a token. + *token = yaml_token_t{ + typ: yaml_SCALAR_TOKEN, + start_mark: start_mark, + end_mark: end_mark, + value: s, + style: yaml_LITERAL_SCALAR_STYLE, + } + if !literal { + token.style = yaml_FOLDED_SCALAR_STYLE + } + return true +} + +// Scan indentation spaces and line breaks for a block scalar. Determine the +// indentation level if needed. +func yaml_parser_scan_block_scalar_breaks(parser *yaml_parser_t, indent *int, breaks *[]byte, start_mark yaml_mark_t, end_mark *yaml_mark_t) bool { + *end_mark = parser.mark + + // Eat the indentation spaces and line breaks. + max_indent := 0 + for { + // Eat the indentation spaces. + if parser.unread < 1 && !yaml_parser_update_buffer(parser, 1) { + return false + } + for (*indent == 0 || parser.mark.column < *indent) && is_space(parser.buffer, parser.buffer_pos) { + skip(parser) + if parser.unread < 1 && !yaml_parser_update_buffer(parser, 1) { + return false + } + } + if parser.mark.column > max_indent { + max_indent = parser.mark.column + } + + // Check for a tab character messing the indentation. + if (*indent == 0 || parser.mark.column < *indent) && is_tab(parser.buffer, parser.buffer_pos) { + return yaml_parser_set_scanner_error(parser, "while scanning a block scalar", + start_mark, "found a tab character where an indentation space is expected") + } + + // Have we found a non-empty line? + if !is_break(parser.buffer, parser.buffer_pos) { + break + } + + // Consume the line break. + if parser.unread < 2 && !yaml_parser_update_buffer(parser, 2) { + return false + } + // [Go] Should really be returning breaks instead. + *breaks = read_line(parser, *breaks) + *end_mark = parser.mark + } + + // Determine the indentation level if needed. + if *indent == 0 { + *indent = max_indent + if *indent < parser.indent+1 { + *indent = parser.indent + 1 + } + if *indent < 1 { + *indent = 1 + } + } + return true +} + +// Scan a quoted scalar. +func yaml_parser_scan_flow_scalar(parser *yaml_parser_t, token *yaml_token_t, single bool) bool { + // Eat the left quote. + start_mark := parser.mark + skip(parser) + + // Consume the content of the quoted scalar. + var s, leading_break, trailing_breaks, whitespaces []byte + for { + // Check that there are no document indicators at the beginning of the line. + if parser.unread < 4 && !yaml_parser_update_buffer(parser, 4) { + return false + } + + if parser.mark.column == 0 && + ((parser.buffer[parser.buffer_pos+0] == '-' && + parser.buffer[parser.buffer_pos+1] == '-' && + parser.buffer[parser.buffer_pos+2] == '-') || + (parser.buffer[parser.buffer_pos+0] == '.' && + parser.buffer[parser.buffer_pos+1] == '.' && + parser.buffer[parser.buffer_pos+2] == '.')) && + is_blankz(parser.buffer, parser.buffer_pos+3) { + yaml_parser_set_scanner_error(parser, "while scanning a quoted scalar", + start_mark, "found unexpected document indicator") + return false + } + + // Check for EOF. + if is_z(parser.buffer, parser.buffer_pos) { + yaml_parser_set_scanner_error(parser, "while scanning a quoted scalar", + start_mark, "found unexpected end of stream") + return false + } + + // Consume non-blank characters. + leading_blanks := false + for !is_blankz(parser.buffer, parser.buffer_pos) { + if single && parser.buffer[parser.buffer_pos] == '\'' && parser.buffer[parser.buffer_pos+1] == '\'' { + // Is is an escaped single quote. + s = append(s, '\'') + skip(parser) + skip(parser) + + } else if single && parser.buffer[parser.buffer_pos] == '\'' { + // It is a right single quote. + break + } else if !single && parser.buffer[parser.buffer_pos] == '"' { + // It is a right double quote. + break + + } else if !single && parser.buffer[parser.buffer_pos] == '\\' && is_break(parser.buffer, parser.buffer_pos+1) { + // It is an escaped line break. + if parser.unread < 3 && !yaml_parser_update_buffer(parser, 3) { + return false + } + skip(parser) + skip_line(parser) + leading_blanks = true + break + + } else if !single && parser.buffer[parser.buffer_pos] == '\\' { + // It is an escape sequence. + code_length := 0 + + // Check the escape character. + switch parser.buffer[parser.buffer_pos+1] { + case '0': + s = append(s, 0) + case 'a': + s = append(s, '\x07') + case 'b': + s = append(s, '\x08') + case 't', '\t': + s = append(s, '\x09') + case 'n': + s = append(s, '\x0A') + case 'v': + s = append(s, '\x0B') + case 'f': + s = append(s, '\x0C') + case 'r': + s = append(s, '\x0D') + case 'e': + s = append(s, '\x1B') + case ' ': + s = append(s, '\x20') + case '"': + s = append(s, '"') + case '\'': + s = append(s, '\'') + case '\\': + s = append(s, '\\') + case 'N': // NEL (#x85) + s = append(s, '\xC2') + s = append(s, '\x85') + case '_': // #xA0 + s = append(s, '\xC2') + s = append(s, '\xA0') + case 'L': // LS (#x2028) + s = append(s, '\xE2') + s = append(s, '\x80') + s = append(s, '\xA8') + case 'P': // PS (#x2029) + s = append(s, '\xE2') + s = append(s, '\x80') + s = append(s, '\xA9') + case 'x': + code_length = 2 + case 'u': + code_length = 4 + case 'U': + code_length = 8 + default: + yaml_parser_set_scanner_error(parser, "while parsing a quoted scalar", + start_mark, "found unknown escape character") + return false + } + + skip(parser) + skip(parser) + + // Consume an arbitrary escape code. + if code_length > 0 { + var value int + + // Scan the character value. + if parser.unread < code_length && !yaml_parser_update_buffer(parser, code_length) { + return false + } + for k := 0; k < code_length; k++ { + if !is_hex(parser.buffer, parser.buffer_pos+k) { + yaml_parser_set_scanner_error(parser, "while parsing a quoted scalar", + start_mark, "did not find expected hexdecimal number") + return false + } + value = (value << 4) + as_hex(parser.buffer, parser.buffer_pos+k) + } + + // Check the value and write the character. + if (value >= 0xD800 && value <= 0xDFFF) || value > 0x10FFFF { + yaml_parser_set_scanner_error(parser, "while parsing a quoted scalar", + start_mark, "found invalid Unicode character escape code") + return false + } + if value <= 0x7F { + s = append(s, byte(value)) + } else if value <= 0x7FF { + s = append(s, byte(0xC0+(value>>6))) + s = append(s, byte(0x80+(value&0x3F))) + } else if value <= 0xFFFF { + s = append(s, byte(0xE0+(value>>12))) + s = append(s, byte(0x80+((value>>6)&0x3F))) + s = append(s, byte(0x80+(value&0x3F))) + } else { + s = append(s, byte(0xF0+(value>>18))) + s = append(s, byte(0x80+((value>>12)&0x3F))) + s = append(s, byte(0x80+((value>>6)&0x3F))) + s = append(s, byte(0x80+(value&0x3F))) + } + + // Advance the pointer. + for k := 0; k < code_length; k++ { + skip(parser) + } + } + } else { + // It is a non-escaped non-blank character. + s = read(parser, s) + } + if parser.unread < 2 && !yaml_parser_update_buffer(parser, 2) { + return false + } + } + + if parser.unread < 1 && !yaml_parser_update_buffer(parser, 1) { + return false + } + + // Check if we are at the end of the scalar. + if single { + if parser.buffer[parser.buffer_pos] == '\'' { + break + } + } else { + if parser.buffer[parser.buffer_pos] == '"' { + break + } + } + + // Consume blank characters. + for is_blank(parser.buffer, parser.buffer_pos) || is_break(parser.buffer, parser.buffer_pos) { + if is_blank(parser.buffer, parser.buffer_pos) { + // Consume a space or a tab character. + if !leading_blanks { + whitespaces = read(parser, whitespaces) + } else { + skip(parser) + } + } else { + if parser.unread < 2 && !yaml_parser_update_buffer(parser, 2) { + return false + } + + // Check if it is a first line break. + if !leading_blanks { + whitespaces = whitespaces[:0] + leading_break = read_line(parser, leading_break) + leading_blanks = true + } else { + trailing_breaks = read_line(parser, trailing_breaks) + } + } + if parser.unread < 1 && !yaml_parser_update_buffer(parser, 1) { + return false + } + } + + // Join the whitespaces or fold line breaks. + if leading_blanks { + // Do we need to fold line breaks? + if len(leading_break) > 0 && leading_break[0] == '\n' { + if len(trailing_breaks) == 0 { + s = append(s, ' ') + } else { + s = append(s, trailing_breaks...) + } + } else { + s = append(s, leading_break...) + s = append(s, trailing_breaks...) + } + trailing_breaks = trailing_breaks[:0] + leading_break = leading_break[:0] + } else { + s = append(s, whitespaces...) + whitespaces = whitespaces[:0] + } + } + + // Eat the right quote. + skip(parser) + end_mark := parser.mark + + // Create a token. + *token = yaml_token_t{ + typ: yaml_SCALAR_TOKEN, + start_mark: start_mark, + end_mark: end_mark, + value: s, + style: yaml_SINGLE_QUOTED_SCALAR_STYLE, + } + if !single { + token.style = yaml_DOUBLE_QUOTED_SCALAR_STYLE + } + return true +} + +// Scan a plain scalar. +func yaml_parser_scan_plain_scalar(parser *yaml_parser_t, token *yaml_token_t) bool { + + var s, leading_break, trailing_breaks, whitespaces []byte + var leading_blanks bool + var indent = parser.indent + 1 + + start_mark := parser.mark + end_mark := parser.mark + + // Consume the content of the plain scalar. + for { + // Check for a document indicator. + if parser.unread < 4 && !yaml_parser_update_buffer(parser, 4) { + return false + } + if parser.mark.column == 0 && + ((parser.buffer[parser.buffer_pos+0] == '-' && + parser.buffer[parser.buffer_pos+1] == '-' && + parser.buffer[parser.buffer_pos+2] == '-') || + (parser.buffer[parser.buffer_pos+0] == '.' && + parser.buffer[parser.buffer_pos+1] == '.' && + parser.buffer[parser.buffer_pos+2] == '.')) && + is_blankz(parser.buffer, parser.buffer_pos+3) { + break + } + + // Check for a comment. + if parser.buffer[parser.buffer_pos] == '#' { + break + } + + // Consume non-blank characters. + for !is_blankz(parser.buffer, parser.buffer_pos) { + + // Check for indicators that may end a plain scalar. + if (parser.buffer[parser.buffer_pos] == ':' && is_blankz(parser.buffer, parser.buffer_pos+1)) || + (parser.flow_level > 0 && + (parser.buffer[parser.buffer_pos] == ',' || + parser.buffer[parser.buffer_pos] == '?' || parser.buffer[parser.buffer_pos] == '[' || + parser.buffer[parser.buffer_pos] == ']' || parser.buffer[parser.buffer_pos] == '{' || + parser.buffer[parser.buffer_pos] == '}')) { + break + } + + // Check if we need to join whitespaces and breaks. + if leading_blanks || len(whitespaces) > 0 { + if leading_blanks { + // Do we need to fold line breaks? + if leading_break[0] == '\n' { + if len(trailing_breaks) == 0 { + s = append(s, ' ') + } else { + s = append(s, trailing_breaks...) + } + } else { + s = append(s, leading_break...) + s = append(s, trailing_breaks...) + } + trailing_breaks = trailing_breaks[:0] + leading_break = leading_break[:0] + leading_blanks = false + } else { + s = append(s, whitespaces...) + whitespaces = whitespaces[:0] + } + } + + // Copy the character. + s = read(parser, s) + + end_mark = parser.mark + if parser.unread < 2 && !yaml_parser_update_buffer(parser, 2) { + return false + } + } + + // Is it the end? + if !(is_blank(parser.buffer, parser.buffer_pos) || is_break(parser.buffer, parser.buffer_pos)) { + break + } + + // Consume blank characters. + if parser.unread < 1 && !yaml_parser_update_buffer(parser, 1) { + return false + } + + for is_blank(parser.buffer, parser.buffer_pos) || is_break(parser.buffer, parser.buffer_pos) { + if is_blank(parser.buffer, parser.buffer_pos) { + + // Check for tab characters that abuse indentation. + if leading_blanks && parser.mark.column < indent && is_tab(parser.buffer, parser.buffer_pos) { + yaml_parser_set_scanner_error(parser, "while scanning a plain scalar", + start_mark, "found a tab character that violates indentation") + return false + } + + // Consume a space or a tab character. + if !leading_blanks { + whitespaces = read(parser, whitespaces) + } else { + skip(parser) + } + } else { + if parser.unread < 2 && !yaml_parser_update_buffer(parser, 2) { + return false + } + + // Check if it is a first line break. + if !leading_blanks { + whitespaces = whitespaces[:0] + leading_break = read_line(parser, leading_break) + leading_blanks = true + } else { + trailing_breaks = read_line(parser, trailing_breaks) + } + } + if parser.unread < 1 && !yaml_parser_update_buffer(parser, 1) { + return false + } + } + + // Check indentation level. + if parser.flow_level == 0 && parser.mark.column < indent { + break + } + } + + // Create a token. + *token = yaml_token_t{ + typ: yaml_SCALAR_TOKEN, + start_mark: start_mark, + end_mark: end_mark, + value: s, + style: yaml_PLAIN_SCALAR_STYLE, + } + + // Note that we change the 'simple_key_allowed' flag. + if leading_blanks { + parser.simple_key_allowed = true + } + return true +} diff --git a/tools/voluspa/vendor/gopkg.in/yaml.v2/sorter.go b/tools/voluspa/vendor/gopkg.in/yaml.v2/sorter.go new file mode 100644 index 00000000000..4c45e660a8f --- /dev/null +++ b/tools/voluspa/vendor/gopkg.in/yaml.v2/sorter.go @@ -0,0 +1,113 @@ +package yaml + +import ( + "reflect" + "unicode" +) + +type keyList []reflect.Value + +func (l keyList) Len() int { return len(l) } +func (l keyList) Swap(i, j int) { l[i], l[j] = l[j], l[i] } +func (l keyList) Less(i, j int) bool { + a := l[i] + b := l[j] + ak := a.Kind() + bk := b.Kind() + for (ak == reflect.Interface || ak == reflect.Ptr) && !a.IsNil() { + a = a.Elem() + ak = a.Kind() + } + for (bk == reflect.Interface || bk == reflect.Ptr) && !b.IsNil() { + b = b.Elem() + bk = b.Kind() + } + af, aok := keyFloat(a) + bf, bok := keyFloat(b) + if aok && bok { + if af != bf { + return af < bf + } + if ak != bk { + return ak < bk + } + return numLess(a, b) + } + if ak != reflect.String || bk != reflect.String { + return ak < bk + } + ar, br := []rune(a.String()), []rune(b.String()) + for i := 0; i < len(ar) && i < len(br); i++ { + if ar[i] == br[i] { + continue + } + al := unicode.IsLetter(ar[i]) + bl := unicode.IsLetter(br[i]) + if al && bl { + return ar[i] < br[i] + } + if al || bl { + return bl + } + var ai, bi int + var an, bn int64 + if ar[i] == '0' || br[i] == '0' { + for j := i-1; j >= 0 && unicode.IsDigit(ar[j]); j-- { + if ar[j] != '0' { + an = 1 + bn = 1 + break + } + } + } + for ai = i; ai < len(ar) && unicode.IsDigit(ar[ai]); ai++ { + an = an*10 + int64(ar[ai]-'0') + } + for bi = i; bi < len(br) && unicode.IsDigit(br[bi]); bi++ { + bn = bn*10 + int64(br[bi]-'0') + } + if an != bn { + return an < bn + } + if ai != bi { + return ai < bi + } + return ar[i] < br[i] + } + return len(ar) < len(br) +} + +// keyFloat returns a float value for v if it is a number/bool +// and whether it is a number/bool or not. +func keyFloat(v reflect.Value) (f float64, ok bool) { + switch v.Kind() { + case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64: + return float64(v.Int()), true + case reflect.Float32, reflect.Float64: + return v.Float(), true + case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64, reflect.Uintptr: + return float64(v.Uint()), true + case reflect.Bool: + if v.Bool() { + return 1, true + } + return 0, true + } + return 0, false +} + +// numLess returns whether a < b. +// a and b must necessarily have the same kind. +func numLess(a, b reflect.Value) bool { + switch a.Kind() { + case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64: + return a.Int() < b.Int() + case reflect.Float32, reflect.Float64: + return a.Float() < b.Float() + case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64, reflect.Uintptr: + return a.Uint() < b.Uint() + case reflect.Bool: + return !a.Bool() && b.Bool() + } + panic("not a number") +} diff --git a/tools/voluspa/vendor/gopkg.in/yaml.v2/writerc.go b/tools/voluspa/vendor/gopkg.in/yaml.v2/writerc.go new file mode 100644 index 00000000000..a2dde608cb7 --- /dev/null +++ b/tools/voluspa/vendor/gopkg.in/yaml.v2/writerc.go @@ -0,0 +1,26 @@ +package yaml + +// Set the writer error and return false. +func yaml_emitter_set_writer_error(emitter *yaml_emitter_t, problem string) bool { + emitter.error = yaml_WRITER_ERROR + emitter.problem = problem + return false +} + +// Flush the output buffer. +func yaml_emitter_flush(emitter *yaml_emitter_t) bool { + if emitter.write_handler == nil { + panic("write handler not set") + } + + // Check if the buffer is empty. + if emitter.buffer_pos == 0 { + return true + } + + if err := emitter.write_handler(emitter, emitter.buffer[:emitter.buffer_pos]); err != nil { + return yaml_emitter_set_writer_error(emitter, "write error: "+err.Error()) + } + emitter.buffer_pos = 0 + return true +} diff --git a/tools/voluspa/vendor/gopkg.in/yaml.v2/yaml.go b/tools/voluspa/vendor/gopkg.in/yaml.v2/yaml.go new file mode 100644 index 00000000000..de85aa4cdb7 --- /dev/null +++ b/tools/voluspa/vendor/gopkg.in/yaml.v2/yaml.go @@ -0,0 +1,466 @@ +// Package yaml implements YAML support for the Go language. +// +// Source code and other details for the project are available at GitHub: +// +// https://github.com/go-yaml/yaml +// +package yaml + +import ( + "errors" + "fmt" + "io" + "reflect" + "strings" + "sync" +) + +// MapSlice encodes and decodes as a YAML map. +// The order of keys is preserved when encoding and decoding. +type MapSlice []MapItem + +// MapItem is an item in a MapSlice. +type MapItem struct { + Key, Value interface{} +} + +// The Unmarshaler interface may be implemented by types to customize their +// behavior when being unmarshaled from a YAML document. The UnmarshalYAML +// method receives a function that may be called to unmarshal the original +// YAML value into a field or variable. It is safe to call the unmarshal +// function parameter more than once if necessary. +type Unmarshaler interface { + UnmarshalYAML(unmarshal func(interface{}) error) error +} + +// The Marshaler interface may be implemented by types to customize their +// behavior when being marshaled into a YAML document. The returned value +// is marshaled in place of the original value implementing Marshaler. +// +// If an error is returned by MarshalYAML, the marshaling procedure stops +// and returns with the provided error. +type Marshaler interface { + MarshalYAML() (interface{}, error) +} + +// Unmarshal decodes the first document found within the in byte slice +// and assigns decoded values into the out value. +// +// Maps and pointers (to a struct, string, int, etc) are accepted as out +// values. If an internal pointer within a struct is not initialized, +// the yaml package will initialize it if necessary for unmarshalling +// the provided data. The out parameter must not be nil. +// +// The type of the decoded values should be compatible with the respective +// values in out. If one or more values cannot be decoded due to a type +// mismatches, decoding continues partially until the end of the YAML +// content, and a *yaml.TypeError is returned with details for all +// missed values. +// +// Struct fields are only unmarshalled if they are exported (have an +// upper case first letter), and are unmarshalled using the field name +// lowercased as the default key. Custom keys may be defined via the +// "yaml" name in the field tag: the content preceding the first comma +// is used as the key, and the following comma-separated options are +// used to tweak the marshalling process (see Marshal). +// Conflicting names result in a runtime error. +// +// For example: +// +// type T struct { +// F int `yaml:"a,omitempty"` +// B int +// } +// var t T +// yaml.Unmarshal([]byte("a: 1\nb: 2"), &t) +// +// See the documentation of Marshal for the format of tags and a list of +// supported tag options. +// +func Unmarshal(in []byte, out interface{}) (err error) { + return unmarshal(in, out, false) +} + +// UnmarshalStrict is like Unmarshal except that any fields that are found +// in the data that do not have corresponding struct members, or mapping +// keys that are duplicates, will result in +// an error. +func UnmarshalStrict(in []byte, out interface{}) (err error) { + return unmarshal(in, out, true) +} + +// A Decorder reads and decodes YAML values from an input stream. +type Decoder struct { + strict bool + parser *parser +} + +// NewDecoder returns a new decoder that reads from r. +// +// The decoder introduces its own buffering and may read +// data from r beyond the YAML values requested. +func NewDecoder(r io.Reader) *Decoder { + return &Decoder{ + parser: newParserFromReader(r), + } +} + +// SetStrict sets whether strict decoding behaviour is enabled when +// decoding items in the data (see UnmarshalStrict). By default, decoding is not strict. +func (dec *Decoder) SetStrict(strict bool) { + dec.strict = strict +} + +// Decode reads the next YAML-encoded value from its input +// and stores it in the value pointed to by v. +// +// See the documentation for Unmarshal for details about the +// conversion of YAML into a Go value. +func (dec *Decoder) Decode(v interface{}) (err error) { + d := newDecoder(dec.strict) + defer handleErr(&err) + node := dec.parser.parse() + if node == nil { + return io.EOF + } + out := reflect.ValueOf(v) + if out.Kind() == reflect.Ptr && !out.IsNil() { + out = out.Elem() + } + d.unmarshal(node, out) + if len(d.terrors) > 0 { + return &TypeError{d.terrors} + } + return nil +} + +func unmarshal(in []byte, out interface{}, strict bool) (err error) { + defer handleErr(&err) + d := newDecoder(strict) + p := newParser(in) + defer p.destroy() + node := p.parse() + if node != nil { + v := reflect.ValueOf(out) + if v.Kind() == reflect.Ptr && !v.IsNil() { + v = v.Elem() + } + d.unmarshal(node, v) + } + if len(d.terrors) > 0 { + return &TypeError{d.terrors} + } + return nil +} + +// Marshal serializes the value provided into a YAML document. The structure +// of the generated document will reflect the structure of the value itself. +// Maps and pointers (to struct, string, int, etc) are accepted as the in value. +// +// Struct fields are only marshalled if they are exported (have an upper case +// first letter), and are marshalled using the field name lowercased as the +// default key. Custom keys may be defined via the "yaml" name in the field +// tag: the content preceding the first comma is used as the key, and the +// following comma-separated options are used to tweak the marshalling process. +// Conflicting names result in a runtime error. +// +// The field tag format accepted is: +// +// `(...) yaml:"[][,[,]]" (...)` +// +// The following flags are currently supported: +// +// omitempty Only include the field if it's not set to the zero +// value for the type or to empty slices or maps. +// Zero valued structs will be omitted if all their public +// fields are zero, unless they implement an IsZero +// method (see the IsZeroer interface type), in which +// case the field will be included if that method returns true. +// +// flow Marshal using a flow style (useful for structs, +// sequences and maps). +// +// inline Inline the field, which must be a struct or a map, +// causing all of its fields or keys to be processed as if +// they were part of the outer struct. For maps, keys must +// not conflict with the yaml keys of other struct fields. +// +// In addition, if the key is "-", the field is ignored. +// +// For example: +// +// type T struct { +// F int `yaml:"a,omitempty"` +// B int +// } +// yaml.Marshal(&T{B: 2}) // Returns "b: 2\n" +// yaml.Marshal(&T{F: 1}} // Returns "a: 1\nb: 0\n" +// +func Marshal(in interface{}) (out []byte, err error) { + defer handleErr(&err) + e := newEncoder() + defer e.destroy() + e.marshalDoc("", reflect.ValueOf(in)) + e.finish() + out = e.out + return +} + +// An Encoder writes YAML values to an output stream. +type Encoder struct { + encoder *encoder +} + +// NewEncoder returns a new encoder that writes to w. +// The Encoder should be closed after use to flush all data +// to w. +func NewEncoder(w io.Writer) *Encoder { + return &Encoder{ + encoder: newEncoderWithWriter(w), + } +} + +// Encode writes the YAML encoding of v to the stream. +// If multiple items are encoded to the stream, the +// second and subsequent document will be preceded +// with a "---" document separator, but the first will not. +// +// See the documentation for Marshal for details about the conversion of Go +// values to YAML. +func (e *Encoder) Encode(v interface{}) (err error) { + defer handleErr(&err) + e.encoder.marshalDoc("", reflect.ValueOf(v)) + return nil +} + +// Close closes the encoder by writing any remaining data. +// It does not write a stream terminating string "...". +func (e *Encoder) Close() (err error) { + defer handleErr(&err) + e.encoder.finish() + return nil +} + +func handleErr(err *error) { + if v := recover(); v != nil { + if e, ok := v.(yamlError); ok { + *err = e.err + } else { + panic(v) + } + } +} + +type yamlError struct { + err error +} + +func fail(err error) { + panic(yamlError{err}) +} + +func failf(format string, args ...interface{}) { + panic(yamlError{fmt.Errorf("yaml: "+format, args...)}) +} + +// A TypeError is returned by Unmarshal when one or more fields in +// the YAML document cannot be properly decoded into the requested +// types. When this error is returned, the value is still +// unmarshaled partially. +type TypeError struct { + Errors []string +} + +func (e *TypeError) Error() string { + return fmt.Sprintf("yaml: unmarshal errors:\n %s", strings.Join(e.Errors, "\n ")) +} + +// -------------------------------------------------------------------------- +// Maintain a mapping of keys to structure field indexes + +// The code in this section was copied from mgo/bson. + +// structInfo holds details for the serialization of fields of +// a given struct. +type structInfo struct { + FieldsMap map[string]fieldInfo + FieldsList []fieldInfo + + // InlineMap is the number of the field in the struct that + // contains an ,inline map, or -1 if there's none. + InlineMap int +} + +type fieldInfo struct { + Key string + Num int + OmitEmpty bool + Flow bool + // Id holds the unique field identifier, so we can cheaply + // check for field duplicates without maintaining an extra map. + Id int + + // Inline holds the field index if the field is part of an inlined struct. + Inline []int +} + +var structMap = make(map[reflect.Type]*structInfo) +var fieldMapMutex sync.RWMutex + +func getStructInfo(st reflect.Type) (*structInfo, error) { + fieldMapMutex.RLock() + sinfo, found := structMap[st] + fieldMapMutex.RUnlock() + if found { + return sinfo, nil + } + + n := st.NumField() + fieldsMap := make(map[string]fieldInfo) + fieldsList := make([]fieldInfo, 0, n) + inlineMap := -1 + for i := 0; i != n; i++ { + field := st.Field(i) + if field.PkgPath != "" && !field.Anonymous { + continue // Private field + } + + info := fieldInfo{Num: i} + + tag := field.Tag.Get("yaml") + if tag == "" && strings.Index(string(field.Tag), ":") < 0 { + tag = string(field.Tag) + } + if tag == "-" { + continue + } + + inline := false + fields := strings.Split(tag, ",") + if len(fields) > 1 { + for _, flag := range fields[1:] { + switch flag { + case "omitempty": + info.OmitEmpty = true + case "flow": + info.Flow = true + case "inline": + inline = true + default: + return nil, errors.New(fmt.Sprintf("Unsupported flag %q in tag %q of type %s", flag, tag, st)) + } + } + tag = fields[0] + } + + if inline { + switch field.Type.Kind() { + case reflect.Map: + if inlineMap >= 0 { + return nil, errors.New("Multiple ,inline maps in struct " + st.String()) + } + if field.Type.Key() != reflect.TypeOf("") { + return nil, errors.New("Option ,inline needs a map with string keys in struct " + st.String()) + } + inlineMap = info.Num + case reflect.Struct: + sinfo, err := getStructInfo(field.Type) + if err != nil { + return nil, err + } + for _, finfo := range sinfo.FieldsList { + if _, found := fieldsMap[finfo.Key]; found { + msg := "Duplicated key '" + finfo.Key + "' in struct " + st.String() + return nil, errors.New(msg) + } + if finfo.Inline == nil { + finfo.Inline = []int{i, finfo.Num} + } else { + finfo.Inline = append([]int{i}, finfo.Inline...) + } + finfo.Id = len(fieldsList) + fieldsMap[finfo.Key] = finfo + fieldsList = append(fieldsList, finfo) + } + default: + //return nil, errors.New("Option ,inline needs a struct value or map field") + return nil, errors.New("Option ,inline needs a struct value field") + } + continue + } + + if tag != "" { + info.Key = tag + } else { + info.Key = strings.ToLower(field.Name) + } + + if _, found = fieldsMap[info.Key]; found { + msg := "Duplicated key '" + info.Key + "' in struct " + st.String() + return nil, errors.New(msg) + } + + info.Id = len(fieldsList) + fieldsList = append(fieldsList, info) + fieldsMap[info.Key] = info + } + + sinfo = &structInfo{ + FieldsMap: fieldsMap, + FieldsList: fieldsList, + InlineMap: inlineMap, + } + + fieldMapMutex.Lock() + structMap[st] = sinfo + fieldMapMutex.Unlock() + return sinfo, nil +} + +// IsZeroer is used to check whether an object is zero to +// determine whether it should be omitted when marshaling +// with the omitempty flag. One notable implementation +// is time.Time. +type IsZeroer interface { + IsZero() bool +} + +func isZero(v reflect.Value) bool { + kind := v.Kind() + if z, ok := v.Interface().(IsZeroer); ok { + if (kind == reflect.Ptr || kind == reflect.Interface) && v.IsNil() { + return true + } + return z.IsZero() + } + switch kind { + case reflect.String: + return len(v.String()) == 0 + case reflect.Interface, reflect.Ptr: + return v.IsNil() + case reflect.Slice: + return v.Len() == 0 + case reflect.Map: + return v.Len() == 0 + case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64: + return v.Int() == 0 + case reflect.Float32, reflect.Float64: + return v.Float() == 0 + case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64, reflect.Uintptr: + return v.Uint() == 0 + case reflect.Bool: + return !v.Bool() + case reflect.Struct: + vt := v.Type() + for i := v.NumField() - 1; i >= 0; i-- { + if vt.Field(i).PkgPath != "" { + continue // Private field + } + if !isZero(v.Field(i)) { + return false + } + } + return true + } + return false +} diff --git a/tools/voluspa/vendor/gopkg.in/yaml.v2/yamlh.go b/tools/voluspa/vendor/gopkg.in/yaml.v2/yamlh.go new file mode 100644 index 00000000000..e25cee563be --- /dev/null +++ b/tools/voluspa/vendor/gopkg.in/yaml.v2/yamlh.go @@ -0,0 +1,738 @@ +package yaml + +import ( + "fmt" + "io" +) + +// The version directive data. +type yaml_version_directive_t struct { + major int8 // The major version number. + minor int8 // The minor version number. +} + +// The tag directive data. +type yaml_tag_directive_t struct { + handle []byte // The tag handle. + prefix []byte // The tag prefix. +} + +type yaml_encoding_t int + +// The stream encoding. +const ( + // Let the parser choose the encoding. + yaml_ANY_ENCODING yaml_encoding_t = iota + + yaml_UTF8_ENCODING // The default UTF-8 encoding. + yaml_UTF16LE_ENCODING // The UTF-16-LE encoding with BOM. + yaml_UTF16BE_ENCODING // The UTF-16-BE encoding with BOM. +) + +type yaml_break_t int + +// Line break types. +const ( + // Let the parser choose the break type. + yaml_ANY_BREAK yaml_break_t = iota + + yaml_CR_BREAK // Use CR for line breaks (Mac style). + yaml_LN_BREAK // Use LN for line breaks (Unix style). + yaml_CRLN_BREAK // Use CR LN for line breaks (DOS style). +) + +type yaml_error_type_t int + +// Many bad things could happen with the parser and emitter. +const ( + // No error is produced. + yaml_NO_ERROR yaml_error_type_t = iota + + yaml_MEMORY_ERROR // Cannot allocate or reallocate a block of memory. + yaml_READER_ERROR // Cannot read or decode the input stream. + yaml_SCANNER_ERROR // Cannot scan the input stream. + yaml_PARSER_ERROR // Cannot parse the input stream. + yaml_COMPOSER_ERROR // Cannot compose a YAML document. + yaml_WRITER_ERROR // Cannot write to the output stream. + yaml_EMITTER_ERROR // Cannot emit a YAML stream. +) + +// The pointer position. +type yaml_mark_t struct { + index int // The position index. + line int // The position line. + column int // The position column. +} + +// Node Styles + +type yaml_style_t int8 + +type yaml_scalar_style_t yaml_style_t + +// Scalar styles. +const ( + // Let the emitter choose the style. + yaml_ANY_SCALAR_STYLE yaml_scalar_style_t = iota + + yaml_PLAIN_SCALAR_STYLE // The plain scalar style. + yaml_SINGLE_QUOTED_SCALAR_STYLE // The single-quoted scalar style. + yaml_DOUBLE_QUOTED_SCALAR_STYLE // The double-quoted scalar style. + yaml_LITERAL_SCALAR_STYLE // The literal scalar style. + yaml_FOLDED_SCALAR_STYLE // The folded scalar style. +) + +type yaml_sequence_style_t yaml_style_t + +// Sequence styles. +const ( + // Let the emitter choose the style. + yaml_ANY_SEQUENCE_STYLE yaml_sequence_style_t = iota + + yaml_BLOCK_SEQUENCE_STYLE // The block sequence style. + yaml_FLOW_SEQUENCE_STYLE // The flow sequence style. +) + +type yaml_mapping_style_t yaml_style_t + +// Mapping styles. +const ( + // Let the emitter choose the style. + yaml_ANY_MAPPING_STYLE yaml_mapping_style_t = iota + + yaml_BLOCK_MAPPING_STYLE // The block mapping style. + yaml_FLOW_MAPPING_STYLE // The flow mapping style. +) + +// Tokens + +type yaml_token_type_t int + +// Token types. +const ( + // An empty token. + yaml_NO_TOKEN yaml_token_type_t = iota + + yaml_STREAM_START_TOKEN // A STREAM-START token. + yaml_STREAM_END_TOKEN // A STREAM-END token. + + yaml_VERSION_DIRECTIVE_TOKEN // A VERSION-DIRECTIVE token. + yaml_TAG_DIRECTIVE_TOKEN // A TAG-DIRECTIVE token. + yaml_DOCUMENT_START_TOKEN // A DOCUMENT-START token. + yaml_DOCUMENT_END_TOKEN // A DOCUMENT-END token. + + yaml_BLOCK_SEQUENCE_START_TOKEN // A BLOCK-SEQUENCE-START token. + yaml_BLOCK_MAPPING_START_TOKEN // A BLOCK-SEQUENCE-END token. + yaml_BLOCK_END_TOKEN // A BLOCK-END token. + + yaml_FLOW_SEQUENCE_START_TOKEN // A FLOW-SEQUENCE-START token. + yaml_FLOW_SEQUENCE_END_TOKEN // A FLOW-SEQUENCE-END token. + yaml_FLOW_MAPPING_START_TOKEN // A FLOW-MAPPING-START token. + yaml_FLOW_MAPPING_END_TOKEN // A FLOW-MAPPING-END token. + + yaml_BLOCK_ENTRY_TOKEN // A BLOCK-ENTRY token. + yaml_FLOW_ENTRY_TOKEN // A FLOW-ENTRY token. + yaml_KEY_TOKEN // A KEY token. + yaml_VALUE_TOKEN // A VALUE token. + + yaml_ALIAS_TOKEN // An ALIAS token. + yaml_ANCHOR_TOKEN // An ANCHOR token. + yaml_TAG_TOKEN // A TAG token. + yaml_SCALAR_TOKEN // A SCALAR token. +) + +func (tt yaml_token_type_t) String() string { + switch tt { + case yaml_NO_TOKEN: + return "yaml_NO_TOKEN" + case yaml_STREAM_START_TOKEN: + return "yaml_STREAM_START_TOKEN" + case yaml_STREAM_END_TOKEN: + return "yaml_STREAM_END_TOKEN" + case yaml_VERSION_DIRECTIVE_TOKEN: + return "yaml_VERSION_DIRECTIVE_TOKEN" + case yaml_TAG_DIRECTIVE_TOKEN: + return "yaml_TAG_DIRECTIVE_TOKEN" + case yaml_DOCUMENT_START_TOKEN: + return "yaml_DOCUMENT_START_TOKEN" + case yaml_DOCUMENT_END_TOKEN: + return "yaml_DOCUMENT_END_TOKEN" + case yaml_BLOCK_SEQUENCE_START_TOKEN: + return "yaml_BLOCK_SEQUENCE_START_TOKEN" + case yaml_BLOCK_MAPPING_START_TOKEN: + return "yaml_BLOCK_MAPPING_START_TOKEN" + case yaml_BLOCK_END_TOKEN: + return "yaml_BLOCK_END_TOKEN" + case yaml_FLOW_SEQUENCE_START_TOKEN: + return "yaml_FLOW_SEQUENCE_START_TOKEN" + case yaml_FLOW_SEQUENCE_END_TOKEN: + return "yaml_FLOW_SEQUENCE_END_TOKEN" + case yaml_FLOW_MAPPING_START_TOKEN: + return "yaml_FLOW_MAPPING_START_TOKEN" + case yaml_FLOW_MAPPING_END_TOKEN: + return "yaml_FLOW_MAPPING_END_TOKEN" + case yaml_BLOCK_ENTRY_TOKEN: + return "yaml_BLOCK_ENTRY_TOKEN" + case yaml_FLOW_ENTRY_TOKEN: + return "yaml_FLOW_ENTRY_TOKEN" + case yaml_KEY_TOKEN: + return "yaml_KEY_TOKEN" + case yaml_VALUE_TOKEN: + return "yaml_VALUE_TOKEN" + case yaml_ALIAS_TOKEN: + return "yaml_ALIAS_TOKEN" + case yaml_ANCHOR_TOKEN: + return "yaml_ANCHOR_TOKEN" + case yaml_TAG_TOKEN: + return "yaml_TAG_TOKEN" + case yaml_SCALAR_TOKEN: + return "yaml_SCALAR_TOKEN" + } + return "" +} + +// The token structure. +type yaml_token_t struct { + // The token type. + typ yaml_token_type_t + + // The start/end of the token. + start_mark, end_mark yaml_mark_t + + // The stream encoding (for yaml_STREAM_START_TOKEN). + encoding yaml_encoding_t + + // The alias/anchor/scalar value or tag/tag directive handle + // (for yaml_ALIAS_TOKEN, yaml_ANCHOR_TOKEN, yaml_SCALAR_TOKEN, yaml_TAG_TOKEN, yaml_TAG_DIRECTIVE_TOKEN). + value []byte + + // The tag suffix (for yaml_TAG_TOKEN). + suffix []byte + + // The tag directive prefix (for yaml_TAG_DIRECTIVE_TOKEN). + prefix []byte + + // The scalar style (for yaml_SCALAR_TOKEN). + style yaml_scalar_style_t + + // The version directive major/minor (for yaml_VERSION_DIRECTIVE_TOKEN). + major, minor int8 +} + +// Events + +type yaml_event_type_t int8 + +// Event types. +const ( + // An empty event. + yaml_NO_EVENT yaml_event_type_t = iota + + yaml_STREAM_START_EVENT // A STREAM-START event. + yaml_STREAM_END_EVENT // A STREAM-END event. + yaml_DOCUMENT_START_EVENT // A DOCUMENT-START event. + yaml_DOCUMENT_END_EVENT // A DOCUMENT-END event. + yaml_ALIAS_EVENT // An ALIAS event. + yaml_SCALAR_EVENT // A SCALAR event. + yaml_SEQUENCE_START_EVENT // A SEQUENCE-START event. + yaml_SEQUENCE_END_EVENT // A SEQUENCE-END event. + yaml_MAPPING_START_EVENT // A MAPPING-START event. + yaml_MAPPING_END_EVENT // A MAPPING-END event. +) + +var eventStrings = []string{ + yaml_NO_EVENT: "none", + yaml_STREAM_START_EVENT: "stream start", + yaml_STREAM_END_EVENT: "stream end", + yaml_DOCUMENT_START_EVENT: "document start", + yaml_DOCUMENT_END_EVENT: "document end", + yaml_ALIAS_EVENT: "alias", + yaml_SCALAR_EVENT: "scalar", + yaml_SEQUENCE_START_EVENT: "sequence start", + yaml_SEQUENCE_END_EVENT: "sequence end", + yaml_MAPPING_START_EVENT: "mapping start", + yaml_MAPPING_END_EVENT: "mapping end", +} + +func (e yaml_event_type_t) String() string { + if e < 0 || int(e) >= len(eventStrings) { + return fmt.Sprintf("unknown event %d", e) + } + return eventStrings[e] +} + +// The event structure. +type yaml_event_t struct { + + // The event type. + typ yaml_event_type_t + + // The start and end of the event. + start_mark, end_mark yaml_mark_t + + // The document encoding (for yaml_STREAM_START_EVENT). + encoding yaml_encoding_t + + // The version directive (for yaml_DOCUMENT_START_EVENT). + version_directive *yaml_version_directive_t + + // The list of tag directives (for yaml_DOCUMENT_START_EVENT). + tag_directives []yaml_tag_directive_t + + // The anchor (for yaml_SCALAR_EVENT, yaml_SEQUENCE_START_EVENT, yaml_MAPPING_START_EVENT, yaml_ALIAS_EVENT). + anchor []byte + + // The tag (for yaml_SCALAR_EVENT, yaml_SEQUENCE_START_EVENT, yaml_MAPPING_START_EVENT). + tag []byte + + // The scalar value (for yaml_SCALAR_EVENT). + value []byte + + // Is the document start/end indicator implicit, or the tag optional? + // (for yaml_DOCUMENT_START_EVENT, yaml_DOCUMENT_END_EVENT, yaml_SEQUENCE_START_EVENT, yaml_MAPPING_START_EVENT, yaml_SCALAR_EVENT). + implicit bool + + // Is the tag optional for any non-plain style? (for yaml_SCALAR_EVENT). + quoted_implicit bool + + // The style (for yaml_SCALAR_EVENT, yaml_SEQUENCE_START_EVENT, yaml_MAPPING_START_EVENT). + style yaml_style_t +} + +func (e *yaml_event_t) scalar_style() yaml_scalar_style_t { return yaml_scalar_style_t(e.style) } +func (e *yaml_event_t) sequence_style() yaml_sequence_style_t { return yaml_sequence_style_t(e.style) } +func (e *yaml_event_t) mapping_style() yaml_mapping_style_t { return yaml_mapping_style_t(e.style) } + +// Nodes + +const ( + yaml_NULL_TAG = "tag:yaml.org,2002:null" // The tag !!null with the only possible value: null. + yaml_BOOL_TAG = "tag:yaml.org,2002:bool" // The tag !!bool with the values: true and false. + yaml_STR_TAG = "tag:yaml.org,2002:str" // The tag !!str for string values. + yaml_INT_TAG = "tag:yaml.org,2002:int" // The tag !!int for integer values. + yaml_FLOAT_TAG = "tag:yaml.org,2002:float" // The tag !!float for float values. + yaml_TIMESTAMP_TAG = "tag:yaml.org,2002:timestamp" // The tag !!timestamp for date and time values. + + yaml_SEQ_TAG = "tag:yaml.org,2002:seq" // The tag !!seq is used to denote sequences. + yaml_MAP_TAG = "tag:yaml.org,2002:map" // The tag !!map is used to denote mapping. + + // Not in original libyaml. + yaml_BINARY_TAG = "tag:yaml.org,2002:binary" + yaml_MERGE_TAG = "tag:yaml.org,2002:merge" + + yaml_DEFAULT_SCALAR_TAG = yaml_STR_TAG // The default scalar tag is !!str. + yaml_DEFAULT_SEQUENCE_TAG = yaml_SEQ_TAG // The default sequence tag is !!seq. + yaml_DEFAULT_MAPPING_TAG = yaml_MAP_TAG // The default mapping tag is !!map. +) + +type yaml_node_type_t int + +// Node types. +const ( + // An empty node. + yaml_NO_NODE yaml_node_type_t = iota + + yaml_SCALAR_NODE // A scalar node. + yaml_SEQUENCE_NODE // A sequence node. + yaml_MAPPING_NODE // A mapping node. +) + +// An element of a sequence node. +type yaml_node_item_t int + +// An element of a mapping node. +type yaml_node_pair_t struct { + key int // The key of the element. + value int // The value of the element. +} + +// The node structure. +type yaml_node_t struct { + typ yaml_node_type_t // The node type. + tag []byte // The node tag. + + // The node data. + + // The scalar parameters (for yaml_SCALAR_NODE). + scalar struct { + value []byte // The scalar value. + length int // The length of the scalar value. + style yaml_scalar_style_t // The scalar style. + } + + // The sequence parameters (for YAML_SEQUENCE_NODE). + sequence struct { + items_data []yaml_node_item_t // The stack of sequence items. + style yaml_sequence_style_t // The sequence style. + } + + // The mapping parameters (for yaml_MAPPING_NODE). + mapping struct { + pairs_data []yaml_node_pair_t // The stack of mapping pairs (key, value). + pairs_start *yaml_node_pair_t // The beginning of the stack. + pairs_end *yaml_node_pair_t // The end of the stack. + pairs_top *yaml_node_pair_t // The top of the stack. + style yaml_mapping_style_t // The mapping style. + } + + start_mark yaml_mark_t // The beginning of the node. + end_mark yaml_mark_t // The end of the node. + +} + +// The document structure. +type yaml_document_t struct { + + // The document nodes. + nodes []yaml_node_t + + // The version directive. + version_directive *yaml_version_directive_t + + // The list of tag directives. + tag_directives_data []yaml_tag_directive_t + tag_directives_start int // The beginning of the tag directives list. + tag_directives_end int // The end of the tag directives list. + + start_implicit int // Is the document start indicator implicit? + end_implicit int // Is the document end indicator implicit? + + // The start/end of the document. + start_mark, end_mark yaml_mark_t +} + +// The prototype of a read handler. +// +// The read handler is called when the parser needs to read more bytes from the +// source. The handler should write not more than size bytes to the buffer. +// The number of written bytes should be set to the size_read variable. +// +// [in,out] data A pointer to an application data specified by +// yaml_parser_set_input(). +// [out] buffer The buffer to write the data from the source. +// [in] size The size of the buffer. +// [out] size_read The actual number of bytes read from the source. +// +// On success, the handler should return 1. If the handler failed, +// the returned value should be 0. On EOF, the handler should set the +// size_read to 0 and return 1. +type yaml_read_handler_t func(parser *yaml_parser_t, buffer []byte) (n int, err error) + +// This structure holds information about a potential simple key. +type yaml_simple_key_t struct { + possible bool // Is a simple key possible? + required bool // Is a simple key required? + token_number int // The number of the token. + mark yaml_mark_t // The position mark. +} + +// The states of the parser. +type yaml_parser_state_t int + +const ( + yaml_PARSE_STREAM_START_STATE yaml_parser_state_t = iota + + yaml_PARSE_IMPLICIT_DOCUMENT_START_STATE // Expect the beginning of an implicit document. + yaml_PARSE_DOCUMENT_START_STATE // Expect DOCUMENT-START. + yaml_PARSE_DOCUMENT_CONTENT_STATE // Expect the content of a document. + yaml_PARSE_DOCUMENT_END_STATE // Expect DOCUMENT-END. + yaml_PARSE_BLOCK_NODE_STATE // Expect a block node. + yaml_PARSE_BLOCK_NODE_OR_INDENTLESS_SEQUENCE_STATE // Expect a block node or indentless sequence. + yaml_PARSE_FLOW_NODE_STATE // Expect a flow node. + yaml_PARSE_BLOCK_SEQUENCE_FIRST_ENTRY_STATE // Expect the first entry of a block sequence. + yaml_PARSE_BLOCK_SEQUENCE_ENTRY_STATE // Expect an entry of a block sequence. + yaml_PARSE_INDENTLESS_SEQUENCE_ENTRY_STATE // Expect an entry of an indentless sequence. + yaml_PARSE_BLOCK_MAPPING_FIRST_KEY_STATE // Expect the first key of a block mapping. + yaml_PARSE_BLOCK_MAPPING_KEY_STATE // Expect a block mapping key. + yaml_PARSE_BLOCK_MAPPING_VALUE_STATE // Expect a block mapping value. + yaml_PARSE_FLOW_SEQUENCE_FIRST_ENTRY_STATE // Expect the first entry of a flow sequence. + yaml_PARSE_FLOW_SEQUENCE_ENTRY_STATE // Expect an entry of a flow sequence. + yaml_PARSE_FLOW_SEQUENCE_ENTRY_MAPPING_KEY_STATE // Expect a key of an ordered mapping. + yaml_PARSE_FLOW_SEQUENCE_ENTRY_MAPPING_VALUE_STATE // Expect a value of an ordered mapping. + yaml_PARSE_FLOW_SEQUENCE_ENTRY_MAPPING_END_STATE // Expect the and of an ordered mapping entry. + yaml_PARSE_FLOW_MAPPING_FIRST_KEY_STATE // Expect the first key of a flow mapping. + yaml_PARSE_FLOW_MAPPING_KEY_STATE // Expect a key of a flow mapping. + yaml_PARSE_FLOW_MAPPING_VALUE_STATE // Expect a value of a flow mapping. + yaml_PARSE_FLOW_MAPPING_EMPTY_VALUE_STATE // Expect an empty value of a flow mapping. + yaml_PARSE_END_STATE // Expect nothing. +) + +func (ps yaml_parser_state_t) String() string { + switch ps { + case yaml_PARSE_STREAM_START_STATE: + return "yaml_PARSE_STREAM_START_STATE" + case yaml_PARSE_IMPLICIT_DOCUMENT_START_STATE: + return "yaml_PARSE_IMPLICIT_DOCUMENT_START_STATE" + case yaml_PARSE_DOCUMENT_START_STATE: + return "yaml_PARSE_DOCUMENT_START_STATE" + case yaml_PARSE_DOCUMENT_CONTENT_STATE: + return "yaml_PARSE_DOCUMENT_CONTENT_STATE" + case yaml_PARSE_DOCUMENT_END_STATE: + return "yaml_PARSE_DOCUMENT_END_STATE" + case yaml_PARSE_BLOCK_NODE_STATE: + return "yaml_PARSE_BLOCK_NODE_STATE" + case yaml_PARSE_BLOCK_NODE_OR_INDENTLESS_SEQUENCE_STATE: + return "yaml_PARSE_BLOCK_NODE_OR_INDENTLESS_SEQUENCE_STATE" + case yaml_PARSE_FLOW_NODE_STATE: + return "yaml_PARSE_FLOW_NODE_STATE" + case yaml_PARSE_BLOCK_SEQUENCE_FIRST_ENTRY_STATE: + return "yaml_PARSE_BLOCK_SEQUENCE_FIRST_ENTRY_STATE" + case yaml_PARSE_BLOCK_SEQUENCE_ENTRY_STATE: + return "yaml_PARSE_BLOCK_SEQUENCE_ENTRY_STATE" + case yaml_PARSE_INDENTLESS_SEQUENCE_ENTRY_STATE: + return "yaml_PARSE_INDENTLESS_SEQUENCE_ENTRY_STATE" + case yaml_PARSE_BLOCK_MAPPING_FIRST_KEY_STATE: + return "yaml_PARSE_BLOCK_MAPPING_FIRST_KEY_STATE" + case yaml_PARSE_BLOCK_MAPPING_KEY_STATE: + return "yaml_PARSE_BLOCK_MAPPING_KEY_STATE" + case yaml_PARSE_BLOCK_MAPPING_VALUE_STATE: + return "yaml_PARSE_BLOCK_MAPPING_VALUE_STATE" + case yaml_PARSE_FLOW_SEQUENCE_FIRST_ENTRY_STATE: + return "yaml_PARSE_FLOW_SEQUENCE_FIRST_ENTRY_STATE" + case yaml_PARSE_FLOW_SEQUENCE_ENTRY_STATE: + return "yaml_PARSE_FLOW_SEQUENCE_ENTRY_STATE" + case yaml_PARSE_FLOW_SEQUENCE_ENTRY_MAPPING_KEY_STATE: + return "yaml_PARSE_FLOW_SEQUENCE_ENTRY_MAPPING_KEY_STATE" + case yaml_PARSE_FLOW_SEQUENCE_ENTRY_MAPPING_VALUE_STATE: + return "yaml_PARSE_FLOW_SEQUENCE_ENTRY_MAPPING_VALUE_STATE" + case yaml_PARSE_FLOW_SEQUENCE_ENTRY_MAPPING_END_STATE: + return "yaml_PARSE_FLOW_SEQUENCE_ENTRY_MAPPING_END_STATE" + case yaml_PARSE_FLOW_MAPPING_FIRST_KEY_STATE: + return "yaml_PARSE_FLOW_MAPPING_FIRST_KEY_STATE" + case yaml_PARSE_FLOW_MAPPING_KEY_STATE: + return "yaml_PARSE_FLOW_MAPPING_KEY_STATE" + case yaml_PARSE_FLOW_MAPPING_VALUE_STATE: + return "yaml_PARSE_FLOW_MAPPING_VALUE_STATE" + case yaml_PARSE_FLOW_MAPPING_EMPTY_VALUE_STATE: + return "yaml_PARSE_FLOW_MAPPING_EMPTY_VALUE_STATE" + case yaml_PARSE_END_STATE: + return "yaml_PARSE_END_STATE" + } + return "" +} + +// This structure holds aliases data. +type yaml_alias_data_t struct { + anchor []byte // The anchor. + index int // The node id. + mark yaml_mark_t // The anchor mark. +} + +// The parser structure. +// +// All members are internal. Manage the structure using the +// yaml_parser_ family of functions. +type yaml_parser_t struct { + + // Error handling + + error yaml_error_type_t // Error type. + + problem string // Error description. + + // The byte about which the problem occurred. + problem_offset int + problem_value int + problem_mark yaml_mark_t + + // The error context. + context string + context_mark yaml_mark_t + + // Reader stuff + + read_handler yaml_read_handler_t // Read handler. + + input_reader io.Reader // File input data. + input []byte // String input data. + input_pos int + + eof bool // EOF flag + + buffer []byte // The working buffer. + buffer_pos int // The current position of the buffer. + + unread int // The number of unread characters in the buffer. + + raw_buffer []byte // The raw buffer. + raw_buffer_pos int // The current position of the buffer. + + encoding yaml_encoding_t // The input encoding. + + offset int // The offset of the current position (in bytes). + mark yaml_mark_t // The mark of the current position. + + // Scanner stuff + + stream_start_produced bool // Have we started to scan the input stream? + stream_end_produced bool // Have we reached the end of the input stream? + + flow_level int // The number of unclosed '[' and '{' indicators. + + tokens []yaml_token_t // The tokens queue. + tokens_head int // The head of the tokens queue. + tokens_parsed int // The number of tokens fetched from the queue. + token_available bool // Does the tokens queue contain a token ready for dequeueing. + + indent int // The current indentation level. + indents []int // The indentation levels stack. + + simple_key_allowed bool // May a simple key occur at the current position? + simple_keys []yaml_simple_key_t // The stack of simple keys. + + // Parser stuff + + state yaml_parser_state_t // The current parser state. + states []yaml_parser_state_t // The parser states stack. + marks []yaml_mark_t // The stack of marks. + tag_directives []yaml_tag_directive_t // The list of TAG directives. + + // Dumper stuff + + aliases []yaml_alias_data_t // The alias data. + + document *yaml_document_t // The currently parsed document. +} + +// Emitter Definitions + +// The prototype of a write handler. +// +// The write handler is called when the emitter needs to flush the accumulated +// characters to the output. The handler should write @a size bytes of the +// @a buffer to the output. +// +// @param[in,out] data A pointer to an application data specified by +// yaml_emitter_set_output(). +// @param[in] buffer The buffer with bytes to be written. +// @param[in] size The size of the buffer. +// +// @returns On success, the handler should return @c 1. If the handler failed, +// the returned value should be @c 0. +// +type yaml_write_handler_t func(emitter *yaml_emitter_t, buffer []byte) error + +type yaml_emitter_state_t int + +// The emitter states. +const ( + // Expect STREAM-START. + yaml_EMIT_STREAM_START_STATE yaml_emitter_state_t = iota + + yaml_EMIT_FIRST_DOCUMENT_START_STATE // Expect the first DOCUMENT-START or STREAM-END. + yaml_EMIT_DOCUMENT_START_STATE // Expect DOCUMENT-START or STREAM-END. + yaml_EMIT_DOCUMENT_CONTENT_STATE // Expect the content of a document. + yaml_EMIT_DOCUMENT_END_STATE // Expect DOCUMENT-END. + yaml_EMIT_FLOW_SEQUENCE_FIRST_ITEM_STATE // Expect the first item of a flow sequence. + yaml_EMIT_FLOW_SEQUENCE_ITEM_STATE // Expect an item of a flow sequence. + yaml_EMIT_FLOW_MAPPING_FIRST_KEY_STATE // Expect the first key of a flow mapping. + yaml_EMIT_FLOW_MAPPING_KEY_STATE // Expect a key of a flow mapping. + yaml_EMIT_FLOW_MAPPING_SIMPLE_VALUE_STATE // Expect a value for a simple key of a flow mapping. + yaml_EMIT_FLOW_MAPPING_VALUE_STATE // Expect a value of a flow mapping. + yaml_EMIT_BLOCK_SEQUENCE_FIRST_ITEM_STATE // Expect the first item of a block sequence. + yaml_EMIT_BLOCK_SEQUENCE_ITEM_STATE // Expect an item of a block sequence. + yaml_EMIT_BLOCK_MAPPING_FIRST_KEY_STATE // Expect the first key of a block mapping. + yaml_EMIT_BLOCK_MAPPING_KEY_STATE // Expect the key of a block mapping. + yaml_EMIT_BLOCK_MAPPING_SIMPLE_VALUE_STATE // Expect a value for a simple key of a block mapping. + yaml_EMIT_BLOCK_MAPPING_VALUE_STATE // Expect a value of a block mapping. + yaml_EMIT_END_STATE // Expect nothing. +) + +// The emitter structure. +// +// All members are internal. Manage the structure using the @c yaml_emitter_ +// family of functions. +type yaml_emitter_t struct { + + // Error handling + + error yaml_error_type_t // Error type. + problem string // Error description. + + // Writer stuff + + write_handler yaml_write_handler_t // Write handler. + + output_buffer *[]byte // String output data. + output_writer io.Writer // File output data. + + buffer []byte // The working buffer. + buffer_pos int // The current position of the buffer. + + raw_buffer []byte // The raw buffer. + raw_buffer_pos int // The current position of the buffer. + + encoding yaml_encoding_t // The stream encoding. + + // Emitter stuff + + canonical bool // If the output is in the canonical style? + best_indent int // The number of indentation spaces. + best_width int // The preferred width of the output lines. + unicode bool // Allow unescaped non-ASCII characters? + line_break yaml_break_t // The preferred line break. + + state yaml_emitter_state_t // The current emitter state. + states []yaml_emitter_state_t // The stack of states. + + events []yaml_event_t // The event queue. + events_head int // The head of the event queue. + + indents []int // The stack of indentation levels. + + tag_directives []yaml_tag_directive_t // The list of tag directives. + + indent int // The current indentation level. + + flow_level int // The current flow level. + + root_context bool // Is it the document root context? + sequence_context bool // Is it a sequence context? + mapping_context bool // Is it a mapping context? + simple_key_context bool // Is it a simple mapping key context? + + line int // The current line. + column int // The current column. + whitespace bool // If the last character was a whitespace? + indention bool // If the last character was an indentation character (' ', '-', '?', ':')? + open_ended bool // If an explicit document end is required? + + // Anchor analysis. + anchor_data struct { + anchor []byte // The anchor value. + alias bool // Is it an alias? + } + + // Tag analysis. + tag_data struct { + handle []byte // The tag handle. + suffix []byte // The tag suffix. + } + + // Scalar analysis. + scalar_data struct { + value []byte // The scalar value. + multiline bool // Does the scalar contain line breaks? + flow_plain_allowed bool // Can the scalar be expessed in the flow plain style? + block_plain_allowed bool // Can the scalar be expressed in the block plain style? + single_quoted_allowed bool // Can the scalar be expressed in the single quoted style? + block_allowed bool // Can the scalar be expressed in the literal or folded styles? + style yaml_scalar_style_t // The output style. + } + + // Dumper stuff + + opened bool // If the stream was already opened? + closed bool // If the stream was already closed? + + // The information associated with the document nodes. + anchors *struct { + references int // The number of references. + anchor int // The anchor id. + serialized bool // If the node has been emitted? + } + + last_anchor_id int // The last assigned anchor id. + + document *yaml_document_t // The currently emitted document. +} diff --git a/tools/voluspa/vendor/gopkg.in/yaml.v2/yamlprivateh.go b/tools/voluspa/vendor/gopkg.in/yaml.v2/yamlprivateh.go new file mode 100644 index 00000000000..8110ce3c37a --- /dev/null +++ b/tools/voluspa/vendor/gopkg.in/yaml.v2/yamlprivateh.go @@ -0,0 +1,173 @@ +package yaml + +const ( + // The size of the input raw buffer. + input_raw_buffer_size = 512 + + // The size of the input buffer. + // It should be possible to decode the whole raw buffer. + input_buffer_size = input_raw_buffer_size * 3 + + // The size of the output buffer. + output_buffer_size = 128 + + // The size of the output raw buffer. + // It should be possible to encode the whole output buffer. + output_raw_buffer_size = (output_buffer_size*2 + 2) + + // The size of other stacks and queues. + initial_stack_size = 16 + initial_queue_size = 16 + initial_string_size = 16 +) + +// Check if the character at the specified position is an alphabetical +// character, a digit, '_', or '-'. +func is_alpha(b []byte, i int) bool { + return b[i] >= '0' && b[i] <= '9' || b[i] >= 'A' && b[i] <= 'Z' || b[i] >= 'a' && b[i] <= 'z' || b[i] == '_' || b[i] == '-' +} + +// Check if the character at the specified position is a digit. +func is_digit(b []byte, i int) bool { + return b[i] >= '0' && b[i] <= '9' +} + +// Get the value of a digit. +func as_digit(b []byte, i int) int { + return int(b[i]) - '0' +} + +// Check if the character at the specified position is a hex-digit. +func is_hex(b []byte, i int) bool { + return b[i] >= '0' && b[i] <= '9' || b[i] >= 'A' && b[i] <= 'F' || b[i] >= 'a' && b[i] <= 'f' +} + +// Get the value of a hex-digit. +func as_hex(b []byte, i int) int { + bi := b[i] + if bi >= 'A' && bi <= 'F' { + return int(bi) - 'A' + 10 + } + if bi >= 'a' && bi <= 'f' { + return int(bi) - 'a' + 10 + } + return int(bi) - '0' +} + +// Check if the character is ASCII. +func is_ascii(b []byte, i int) bool { + return b[i] <= 0x7F +} + +// Check if the character at the start of the buffer can be printed unescaped. +func is_printable(b []byte, i int) bool { + return ((b[i] == 0x0A) || // . == #x0A + (b[i] >= 0x20 && b[i] <= 0x7E) || // #x20 <= . <= #x7E + (b[i] == 0xC2 && b[i+1] >= 0xA0) || // #0xA0 <= . <= #xD7FF + (b[i] > 0xC2 && b[i] < 0xED) || + (b[i] == 0xED && b[i+1] < 0xA0) || + (b[i] == 0xEE) || + (b[i] == 0xEF && // #xE000 <= . <= #xFFFD + !(b[i+1] == 0xBB && b[i+2] == 0xBF) && // && . != #xFEFF + !(b[i+1] == 0xBF && (b[i+2] == 0xBE || b[i+2] == 0xBF)))) +} + +// Check if the character at the specified position is NUL. +func is_z(b []byte, i int) bool { + return b[i] == 0x00 +} + +// Check if the beginning of the buffer is a BOM. +func is_bom(b []byte, i int) bool { + return b[0] == 0xEF && b[1] == 0xBB && b[2] == 0xBF +} + +// Check if the character at the specified position is space. +func is_space(b []byte, i int) bool { + return b[i] == ' ' +} + +// Check if the character at the specified position is tab. +func is_tab(b []byte, i int) bool { + return b[i] == '\t' +} + +// Check if the character at the specified position is blank (space or tab). +func is_blank(b []byte, i int) bool { + //return is_space(b, i) || is_tab(b, i) + return b[i] == ' ' || b[i] == '\t' +} + +// Check if the character at the specified position is a line break. +func is_break(b []byte, i int) bool { + return (b[i] == '\r' || // CR (#xD) + b[i] == '\n' || // LF (#xA) + b[i] == 0xC2 && b[i+1] == 0x85 || // NEL (#x85) + b[i] == 0xE2 && b[i+1] == 0x80 && b[i+2] == 0xA8 || // LS (#x2028) + b[i] == 0xE2 && b[i+1] == 0x80 && b[i+2] == 0xA9) // PS (#x2029) +} + +func is_crlf(b []byte, i int) bool { + return b[i] == '\r' && b[i+1] == '\n' +} + +// Check if the character is a line break or NUL. +func is_breakz(b []byte, i int) bool { + //return is_break(b, i) || is_z(b, i) + return ( // is_break: + b[i] == '\r' || // CR (#xD) + b[i] == '\n' || // LF (#xA) + b[i] == 0xC2 && b[i+1] == 0x85 || // NEL (#x85) + b[i] == 0xE2 && b[i+1] == 0x80 && b[i+2] == 0xA8 || // LS (#x2028) + b[i] == 0xE2 && b[i+1] == 0x80 && b[i+2] == 0xA9 || // PS (#x2029) + // is_z: + b[i] == 0) +} + +// Check if the character is a line break, space, or NUL. +func is_spacez(b []byte, i int) bool { + //return is_space(b, i) || is_breakz(b, i) + return ( // is_space: + b[i] == ' ' || + // is_breakz: + b[i] == '\r' || // CR (#xD) + b[i] == '\n' || // LF (#xA) + b[i] == 0xC2 && b[i+1] == 0x85 || // NEL (#x85) + b[i] == 0xE2 && b[i+1] == 0x80 && b[i+2] == 0xA8 || // LS (#x2028) + b[i] == 0xE2 && b[i+1] == 0x80 && b[i+2] == 0xA9 || // PS (#x2029) + b[i] == 0) +} + +// Check if the character is a line break, space, tab, or NUL. +func is_blankz(b []byte, i int) bool { + //return is_blank(b, i) || is_breakz(b, i) + return ( // is_blank: + b[i] == ' ' || b[i] == '\t' || + // is_breakz: + b[i] == '\r' || // CR (#xD) + b[i] == '\n' || // LF (#xA) + b[i] == 0xC2 && b[i+1] == 0x85 || // NEL (#x85) + b[i] == 0xE2 && b[i+1] == 0x80 && b[i+2] == 0xA8 || // LS (#x2028) + b[i] == 0xE2 && b[i+1] == 0x80 && b[i+2] == 0xA9 || // PS (#x2029) + b[i] == 0) +} + +// Determine the width of the character. +func width(b byte) int { + // Don't replace these by a switch without first + // confirming that it is being inlined. + if b&0x80 == 0x00 { + return 1 + } + if b&0xE0 == 0xC0 { + return 2 + } + if b&0xF0 == 0xE0 { + return 3 + } + if b&0xF8 == 0xF0 { + return 4 + } + return 0 + +} diff --git a/tools/voluspa/version.go b/tools/voluspa/version.go new file mode 100644 index 00000000000..97ea06e872e --- /dev/null +++ b/tools/voluspa/version.go @@ -0,0 +1,28 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package voluspa + +var ( + // Version is the build version + Version = "dev" + + // GitVersion is the Git revision hash + GitVersion = "dev" +) diff --git a/tools/voluspa/version/version.go b/tools/voluspa/version/version.go new file mode 100644 index 00000000000..cc71dc4f65d --- /dev/null +++ b/tools/voluspa/version/version.go @@ -0,0 +1,39 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package version + +import "expvar" + +var ( + // Version is the build version + Version = "dev" + + // GitVersion is the Git revision hash + GitVersion = "dev" +) + +func init() { + if len(GitVersion) > 0 { + Version = Version + "/" + GitVersion + } + + v := expvar.NewString("version") + v.Set(Version) +} diff --git a/tools/voluspa/version/version_test.go b/tools/voluspa/version/version_test.go new file mode 100644 index 00000000000..701d5109d2d --- /dev/null +++ b/tools/voluspa/version/version_test.go @@ -0,0 +1,20 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package version diff --git a/tools/voluspa/voluspa.go b/tools/voluspa/voluspa.go new file mode 100644 index 00000000000..2de67d2118f --- /dev/null +++ b/tools/voluspa/voluspa.go @@ -0,0 +1,330 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package voluspa + +import ( + "bytes" + "encoding/json" + "errors" + "fmt" + "io/ioutil" + "sort" + "strconv" + "strings" + + "github.com/xeipuuv/gojsonschema" + yaml "gopkg.in/yaml.v2" +) + +var ( + ErrMinimumConfigsNotMet = errors.New("Need at least 1 parsed config file") +) + +type Voluspa struct { + parsedConfigs []*CustomerConfig + + installLocation string + options *Options + defaultSchemaVersion int +} + +func NewVoluspa() (*Voluspa, error) { + return NewVoluspaWithOptions(&Options{}, DefaultTrafficserverConfigurationDir) +} + +func NewVoluspaWithOptions(options *Options, installLocation string) (*Voluspa, error) { + return &Voluspa{ + installLocation: installLocation, + options: options, + defaultSchemaVersion: 1, + }, nil +} + +func (v *Voluspa) SchemaDefaultVersion() int { + return v.defaultSchemaVersion +} + +func (v *Voluspa) SchemaDefinition(version int) ([]byte, error) { + if version == 0 { + version = v.SchemaDefaultVersion() + } + + var filename string + if len(v.options.SchemaLocation) > 0 { + filename = fmt.Sprintf("%s/schema_v%d.json", v.options.SchemaLocation, version) + } else { + filename = fmt.Sprintf("schema_v%d.json", version) + } + + return ioutil.ReadFile(filename) +} + +func (v *Voluspa) validateSchema(contents []byte) error { + var ycfg interface{} + if err := yaml.Unmarshal(contents, &ycfg); err != nil { + return err + } + + ycfg = convert(ycfg) + + jcfg, err := json.Marshal(ycfg) + if err != nil { + return err + } + + schemaVersion := v.SchemaDefaultVersion() // TODO extract schema_version from ycfg + schemaContent, err := v.SchemaDefinition(schemaVersion) + if err != nil { + return err + } + + schema, err := gojsonschema.NewSchema(gojsonschema.NewBytesLoader(schemaContent)) + if err != nil { + return fmt.Errorf("error loading the version %d schema definition: %v", schemaVersion, err) + } + + result, err := schema.Validate(gojsonschema.NewBytesLoader(jcfg)) + if err != nil { + return err + } + + if result.Valid() { + return nil + } + + var buf bytes.Buffer + for _, desc := range result.Errors() { + buf.WriteString(fmt.Sprintf(" - %s\n", desc)) + } + + return fmt.Errorf("\n%s", buf.String()) +} + +func convert(i interface{}) interface{} { + switch x := i.(type) { + case map[interface{}]interface{}: + m2 := map[string]interface{}{} + for k, v := range x { + var k2 string + switch k := k.(type) { + case int: + k2 = strconv.Itoa(k) + default: + k2 = k.(string) + } + m2[k2] = convert(v) + } + return m2 + case []interface{}: + for i, v := range x { + x[i] = convert(v) + } + } + return i +} + +// AddConfigByBytes takes a []byte buffer containing a Voluspa YAML config file and a filename and processes it +func (v *Voluspa) AddConfigByBytes(buffer []byte, filename string) error { + if !v.options.SkipSchemaValidation { + if err := v.validateSchema(buffer); err != nil { + return wrapError(filename, err) + } + } + + cfg, err := NewPropertyConfigWithDefaults(buffer, v.options.DefaultsLocation) + if err != nil { + return wrapError(filename, err) + } + + cc, err := NewCustomerConfig(cfg, filename, *v.options) + if err != nil { + return wrapError(filename, err) + } + + v.parsedConfigs = append(v.parsedConfigs, cc) + + return nil +} + +func (v *Voluspa) AddConfig(filename string) error { + buffer, err := ioutil.ReadFile(filename) + if err != nil { + return err + } + return v.AddConfigByBytes(buffer, filename) +} + +func (v *Voluspa) filterConfigsByCDNAndRole(cdns, roles []string) []*CustomerConfig { + if len(cdns) == 0 { + return v.parsedConfigs + } + + cdnMap := make(map[string]interface{}) + for k := range cdns { + cdnMap[cdns[k]] = nil + } + + roleMap := make(map[string]interface{}) + for k := range roles { + roleMap[roles[k]] = nil + } + + var grouped []*CustomerConfig + for _, config := range v.parsedConfigs { + if config.lifecycle == Retired { + continue + } + + if !v.options.PromoteRolesToCDN && isPromotableRole(config.role) { + continue + } + + includeCDN := false + for cdn := range config.cdn { + if _, exists := cdnMap[cdn]; exists { + includeCDN = true + break + } + } + + var includeRole bool + if len(config.role) > 0 { + _, includeRole = roleMap[config.role] + } else { + includeRole = true + } + + if includeCDN && includeRole { + grouped = append(grouped, config) + } + } + + sort.Stable(sort.Reverse(byQPSAndName(grouped))) + + return grouped +} + +// Validate will run all validation methods and return an error if any issues were found +// strict will enable extra checks that are known to fail with reasonable sets of configuration files +func (v *Voluspa) Validate(strict bool) error { + var errs Errors + errs = append(errs, v.ensurePropertyNameUniqueness()...) + errs = append(errs, v.ensureAliasUniqueness()...) + errs = append(errs, v.validateRegexOptions()...) + errs = append(errs, v.ensureValidSchemes()...) + + if strict { + errs = append(errs, v.ensureHostsResolve()...) + } + + if len(errs) > 0 { + return errs + } + + return nil +} + +// WriteAllFiles will write out all files, filter by passed in cdns and roles +func (v *Voluspa) WriteAllFiles(cdns, roles []string) error { + allFiles, err := v.GetManagedFilesByCDNAndRole(cdns, roles) + if err != nil { + return err + } + + writer, err := NewFilesystemWriter(v.options) + if err != nil { + return err + } + + return writer.WriteFiles(allFiles) +} + +// PropertyCount returns the number of properties +func (v *Voluspa) PropertyCount() int { + return len(v.parsedConfigs) +} + +type configGenerator interface { + // Do processes parsedConfigs, optionally merging, returning a slice of ManagedFiles + Do(parsedConfigs []*CustomerConfig, merge bool) ([]ManagedFile, error) +} + +func getGenerators(options *Options) []configGenerator { + var cg []configGenerator + cg = append(cg, newPropertyRemapGenerator(options)) + cg = append(cg, newTopLevelRemapGenerator(options)) + cg = append(cg, newParentConfigurator(options)) + cg = append(cg, newSSLMulticertConfigurator(options)) + cg = append(cg, &HAProxyConfigGenerator{}) + cg = append(cg, newHostingConfigurator(options)) + return cg +} + +func (v *Voluspa) GetManagedFiles(cdns []string) ([]ManagedFile, error) { + return v.GetManagedFilesByCDNAndRole(cdns, nil) +} + +func (v *Voluspa) GetManagedFilesByCDNAndRole(cdns, roles []string) ([]ManagedFile, error) { + cc := v.filterConfigsByCDNAndRole(cdns, roles) + merged := false + if len(cdns) > 0 { + merged = true + } + + var allFiles []ManagedFile + for _, g := range getGenerators(v.options) { + managedFiles, err := g.Do(cc, merged) + if err != nil { + return nil, err + } + allFiles = append(allFiles, managedFiles...) + } + return allFiles, nil +} + +func wrapError(filename string, err error) error { + _, ok := err.(Errors) + if !ok { + return fmt.Errorf("problem loading %q: %s", filename, err) + } + return fmt.Errorf("problem loading %q:\n%s", filename, err) +} + +// byQPSAndName implements sort.Interface, sorting by a CustomerConfig by qps/name. +type byQPSAndName []*CustomerConfig + +func (s byQPSAndName) Len() int { return len(s) } +func (s byQPSAndName) Swap(i, j int) { s[i], s[j] = s[j], s[i] } +func (s byQPSAndName) Less(i, j int) bool { + if s[i].qps < s[j].qps { + return true + } + if s[i].qps > s[j].qps { + return false + } + + // normalize the property names for sorting + return strings.ToLower(s[i].property) > strings.ToLower(s[j].property) +} + +// isPromotableRole returns true of role is promotable +func isPromotableRole(role string) bool { + return strings.HasPrefix(role, "roles_") +}