docs: Categorize transforms into functions or runtimes

.meta/.schema.json

@@ -440,6 +440,7 @@
         "common",
         "function_category",
         "requirements",
+        "sub_type",
         "title"
       ],
       "properties": {
@@ -503,6 +504,14 @@
           "$ref": "#/definitions/requirements",
           "description": "Component requirements"
         },
+        "sub_type": {
+          "type": "string",
+          "description": "The type of transform. Sub-type since `type` is used to describe the component type (source, transform, or sink).",
+          "enum": [
+            "function",
+            "runtime"
+          ]
+        },
         "title": {
           "type": "string",
           "description": "A simple title for this transform, typically one word.",

.meta/transforms/add_fields.toml

@@ -7,6 +7,7 @@ function_category = "shape"
 input_types = ["log"]
 output_types = ["log"]
 requirements = {}
+sub_type = "function"
 
 <%= render("_partials/_component_options.toml", type: "transform", name: "add_fields") %>
 

.meta/transforms/add_tags.toml

@@ -7,6 +7,7 @@ function_category = "shape"
 input_types = ["metric"]
 output_types = ["metric"]
 requirements = {}
+sub_type = "function"
 
 <%= render("_partials/_component_options.toml", type: "transform", name: "add_tags") %>
 

.meta/transforms/ansi_stripper.toml

@@ -7,6 +7,7 @@ function_category = "sanitize"
 input_types = ["log"]
 output_types = ["log"]
 requirements = {}
+sub_type = "function"
 
 <%= render("_partials/_component_options.toml", type: "transform", name: "ansi_stripper") %>
 

.meta/transforms/aws_ec2_metadata.toml

@@ -7,6 +7,7 @@ function_category = "enrich"
 input_types = ["log"]
 output_types = ["log"]
 requirements.network = true
+sub_type = "function"
 
 <%= render("_partials/_component_options.toml", type: "transform", name: "aws_ec2_metadata") %>
 

.meta/transforms/coercer.toml

@@ -7,6 +7,7 @@ function_category = "parse"
 input_types = ["log"]
 output_types = ["log"]
 requirements = {}
+sub_type = "function"
 
 <%= render("_partials/_component_options.toml", type: "transform", name: "coercer") %>
 

.meta/transforms/concat.toml

@@ -7,6 +7,7 @@ function_category = "filter"
 input_types = ["log"]
 output_types = ["log"]
 requirements = {}
+sub_type = "function"
 
 <%= render("_partials/_component_options.toml", type: "transform", name: "concat") %>
 

.meta/transforms/field_filter.toml

@@ -7,6 +7,7 @@ function_category = "filter"
 input_types = ["log"]
 output_types = ["log"]
 requirements = {}
+sub_type = "function"
 
 <%= render("_partials/_component_options.toml", type: "transform", name: "field_filter") %>
 

.meta/transforms/geoip.toml

@@ -7,6 +7,7 @@ function_category = "enrich"
 input_types = ["log"]
 output_types = ["log"]
 requirements = {}
+sub_type = "function"
 
 <%= render("_partials/_component_options.toml", type: "transform", name: "geoip") %>
 

.meta/transforms/grok_parser.toml

@@ -7,6 +7,7 @@ function_category = "parse"
 input_types = ["log"]
 output_types = ["log"]
 requirements = {}
+sub_type = "function"
 
 <%= render("_partials/_component_options.toml", type: "transform", name: "grok_parser") %>
 

.meta/transforms/json_parser.toml

@@ -7,6 +7,7 @@ function_category = "parse"
 input_types = ["log"]
 output_types = ["log"]
 requirements = {}
+sub_type = "function"
 
 <%= render("_partials/_component_options.toml", type: "transform", name: "json_parser") %>
 

.meta/transforms/log_to_metric.toml

@@ -7,6 +7,7 @@ function_category = "convert"
 input_types = ["log"]
 output_types = ["metric"]
 requirements = {}
+sub_type = "function"
 
 <%= render("_partials/_component_options.toml", type: "transform", name: "log_to_metric") %>
 

.meta/transforms/logfmt_parser.toml

@@ -9,6 +9,7 @@ function_category = "parse"
 input_types = ["log"]
 output_types = ["log"]
 requirements = {}
+sub_type = "function"
 
 <%= render("_partials/_component_options.toml", type: "transform", name: "logfmt_parser") %>
 

.meta/transforms/lua.toml

@@ -7,6 +7,7 @@ function_category = "program"
 input_types = ["log"]
 output_types = ["log"]
 requirements = {}
+sub_type = "runtime"
 
 <%= render("_partials/_component_options.toml", type: "transform", name: "lua") %>
 

.meta/transforms/merge.toml

@@ -7,6 +7,7 @@ function_category = "aggregate"
 input_types = ["log"]
 output_types = ["log"]
 requirements = {}
+sub_type = "function"
 
 <%= render("_partials/_component_options.toml", type: "transform", name: "merge") %>
 

.meta/transforms/regex_parser.toml

@@ -9,6 +9,7 @@ function_category = "parse"
 input_types = ["log"]
 output_types = ["log"]
 requirements = {}
+sub_type = "function"
 
 <%= render("_partials/_component_options.toml", type: "transform", name: "regex_parser") %>
 

.meta/transforms/remove_fields.toml

@@ -7,6 +7,7 @@ function_category = "shape"
 input_types = ["log"]
 output_types = ["log"]
 requirements = {}
+sub_type = "function"
 
 <%= render("_partials/_component_options.toml", type: "transform", name: "remove_fields") %>
 

.meta/transforms/remove_tags.toml

@@ -7,6 +7,7 @@ function_category = "shape"
 input_types = ["metric"]
 output_types = ["metric"]
 requirements = {}
+sub_type = "function"
 
 <%= render("_partials/_component_options.toml", type: "transform", name: "remove_tags") %>
 

.meta/transforms/rename_fields.toml

@@ -7,6 +7,7 @@ function_category = "shape"
 input_types = ["log"]
 output_types = ["log"]
 requirements = {}
+sub_type = "function"
 
 <%= render("_partials/_component_options.toml", type: "transform", name: "rename_fields") %>
 

.meta/transforms/sampler.toml

@@ -7,6 +7,7 @@ function_category = "filter"
 input_types = ["log"]
 output_types = ["log"]
 requirements = {}
+sub_type = "function"
 
 <%= render("_partials/_component_options.toml", type: "transform", name: "sampler") %>
 

.meta/transforms/split.toml

@@ -10,6 +10,7 @@ function_category = "parse"
 input_types = ["log"]
 output_types = ["log"]
 requirements = {}
+sub_type = "function"
 
 <%= render("_partials/_component_options.toml", type: "transform", name: "split") %>
 

.meta/transforms/swimlanes.toml

@@ -7,6 +7,7 @@ function_category = "filter"
 input_types = ["log"]
 output_types = ["log"]
 requirements = {}
+sub_type = "function"
 
 <%= render("_partials/_component_options.toml", type: "transform", name: "swimlanes") %>
 

.meta/transforms/tokenizer.toml

@@ -10,6 +10,7 @@ function_category = "parse"
 input_types = ["log"]
 output_types = ["log"]
 requirements = {}
+sub_type = "function"
 
 <%= render("_partials/_component_options.toml", type: "transform", name: "tokenizer") %>
 

README.md

@@ -67,7 +67,7 @@ pipelines. Compared to Logstash and friends, Vector [improves throughput by
 #### Reference
 
 * [**Sources**][docs.sources] - [docker][docs.sources.docker], [file][docs.sources.file], [http][docs.sources.http], [journald][docs.sources.journald], [kafka][docs.sources.kafka], [socket][docs.sources.socket], and [7 more...][docs.sources]
-* [**Transforms**][docs.transforms] - [json_parser][docs.transforms.json_parser], [log_to_metric][docs.transforms.log_to_metric], [logfmt_parser][docs.transforms.logfmt_parser], [lua][docs.transforms.lua], [regex_parser][docs.transforms.regex_parser], [sampler][docs.transforms.sampler], and [16 more...][docs.transforms]
+* [**Transforms**][docs.transforms] - [json_parser][docs.transforms.functions.json_parser], [log_to_metric][docs.transforms.functions.log_to_metric], [logfmt_parser][docs.transforms.functions.logfmt_parser], [lua][docs.transforms.runtimes.lua], [regex_parser][docs.transforms.functions.regex_parser], [sampler][docs.transforms.functions.sampler], and [16 more...][docs.transforms]
 * [**Sinks**][docs.sinks] - [aws_cloudwatch_logs][docs.sinks.aws_cloudwatch_logs], [aws_s3][docs.sinks.aws_s3], [clickhouse][docs.sinks.clickhouse], [elasticsearch][docs.sinks.elasticsearch], [gcp_cloud_storage][docs.sinks.gcp_cloud_storage], [gcp_pubsub][docs.sinks.gcp_pubsub], and [21 more...][docs.sinks]
 
 #### Administration
@@ -131,15 +131,15 @@ To learn more about our correctness tests, please see the [Vector test harness][
 #### Control Flow
 
 * **Pipelining** - A [directed acyclic graph processing model][docs.configuration] allows for flexible topologies.
-* **Control-flow** - Transforms like the [`swimlanes` transform][docs.transforms.swimlanes] allow for complex control-flow logic.
+* **Control-flow** - Transforms like the [`swimlanes` transform][docs.transforms.functions.swimlanes] allow for complex control-flow logic.
 * **Dynamic partitioning** - Create [dynamic partitions on the fly][docs.sinks.aws_s3#partitioning] with Vector's [templating syntax][docs.reference.templating].
 
 #### Data Processing
 
-* **Programmable transforms** - [Lua][docs.transforms.lua], [Javascript (coming soon)][urls.pr_721], and [WASM (coming soon)][urls.issue_1802] transforms.
-* **Rich parsing** - [Regex][docs.transforms.regex_parser], [Grok][docs.transforms.grok_parser], and [more][urls.vector_parsing_transforms] allow for rich parsing.
+* **Programmable transforms** - [Lua][docs.transforms.runtimes.lua], [Javascript (coming soon)][urls.pr_721], and [WASM (coming soon)][urls.issue_1802] transforms.
+* **Rich parsing** - [Regex][docs.transforms.functions.regex_parser], [Grok][docs.transforms.functions.grok_parser], and [more][urls.vector_parsing_transforms] allow for rich parsing.
 * **Context enrichment** - [Enrich data with environment context][urls.vector_enriching_transforms].
-* **Metrics derivation** - [Derive logs from metrics][docs.transforms.log_to_metric].
+* **Metrics derivation** - [Derive logs from metrics][docs.transforms.functions.log_to_metric].
 * **Multi-line merging** - [Merge multi-line logs][docs.sources.file#multiline] into one event, such as stacktraces.
 
 #### Operations
@@ -248,14 +248,14 @@ Or use your own [preferred method][docs.installation].
 [docs.sources.socket]: https://vector.dev/docs/reference/sources/socket/
 [docs.sources]: https://vector.dev/docs/reference/sources/
 [docs.topologies]: https://vector.dev/docs/setup/deployment/topologies/
-[docs.transforms.grok_parser]: https://vector.dev/docs/reference/transforms/grok_parser/
-[docs.transforms.json_parser]: https://vector.dev/docs/reference/transforms/json_parser/
-[docs.transforms.log_to_metric]: https://vector.dev/docs/reference/transforms/log_to_metric/
-[docs.transforms.logfmt_parser]: https://vector.dev/docs/reference/transforms/logfmt_parser/
-[docs.transforms.lua]: https://vector.dev/docs/reference/transforms/lua/
-[docs.transforms.regex_parser]: https://vector.dev/docs/reference/transforms/regex_parser/
-[docs.transforms.sampler]: https://vector.dev/docs/reference/transforms/sampler/
-[docs.transforms.swimlanes]: https://vector.dev/docs/reference/transforms/swimlanes/
+[docs.transforms.functions.grok_parser]: https://vector.dev/docs/reference/transforms/functions/grok_parser/
+[docs.transforms.functions.json_parser]: https://vector.dev/docs/reference/transforms/functions/json_parser/
+[docs.transforms.functions.log_to_metric]: https://vector.dev/docs/reference/transforms/functions/log_to_metric/
+[docs.transforms.functions.logfmt_parser]: https://vector.dev/docs/reference/transforms/functions/logfmt_parser/
+[docs.transforms.functions.regex_parser]: https://vector.dev/docs/reference/transforms/functions/regex_parser/
+[docs.transforms.functions.sampler]: https://vector.dev/docs/reference/transforms/functions/sampler/
+[docs.transforms.functions.swimlanes]: https://vector.dev/docs/reference/transforms/functions/swimlanes/
+[docs.transforms.runtimes.lua]: https://vector.dev/docs/reference/transforms/runtimes/lua/
 [docs.transforms]: https://vector.dev/docs/reference/transforms/
 [docs.updating]: https://vector.dev/docs/administration/updating/
 [docs.validating]: https://vector.dev/docs/administration/validating/

README.md.erb

@@ -109,15 +109,15 @@ To learn more about our correctness tests, please see the [Vector test harness][
 #### Control Flow
 
 * **Pipelining** - A [directed acyclic graph processing model][docs.configuration] allows for flexible topologies.
-* **Control-flow** - Transforms like the [`swimlanes` transform][docs.transforms.swimlanes] allow for complex control-flow logic.
+* **Control-flow** - Transforms like the [`swimlanes` transform][docs.transforms.functions.swimlanes] allow for complex control-flow logic.
 * **Dynamic partitioning** - Create [dynamic partitions on the fly][docs.sinks.aws_s3#partitioning] with Vector's [templating syntax][docs.reference.templating].
 
 #### Data Processing
 
-* **Programmable transforms** - [Lua][docs.transforms.lua], [Javascript (coming soon)][urls.pr_721], and [WASM (coming soon)][urls.issue_1802] transforms.
-* **Rich parsing** - [Regex][docs.transforms.regex_parser], [Grok][docs.transforms.grok_parser], and [more][urls.vector_parsing_transforms] allow for rich parsing.
+* **Programmable transforms** - [Lua][docs.transforms.runtimes.lua], [Javascript (coming soon)][urls.pr_721], and [WASM (coming soon)][urls.issue_1802] transforms.
+* **Rich parsing** - [Regex][docs.transforms.functions.regex_parser], [Grok][docs.transforms.functions.grok_parser], and [more][urls.vector_parsing_transforms] allow for rich parsing.
 * **Context enrichment** - [Enrich data with environment context][urls.vector_enriching_transforms].
-* **Metrics derivation** - [Derive logs from metrics][docs.transforms.log_to_metric].
+* **Metrics derivation** - [Derive logs from metrics][docs.transforms.functions.log_to_metric].
 * **Multi-line merging** - [Merge multi-line logs][docs.sources.file#multiline] into one event, such as stacktraces.
 
 #### Operations

scripts/generate.rb

@@ -197,24 +197,25 @@ def url_valid?(url)
 # Create missing component templates
 #
 
-metadata.components.each do |component|
-  template_path = "#{REFERENCE_ROOT}/#{component.type.pluralize}/#{component.name}.md.erb"
+# metadata.components.each do |component|
+#   template_path = "#{REFERENCE_ROOT}/#{component.type.pluralize}/#{component.name}.md.erb"
 
-  if !File.exists?(template_path)
-    contents = templates.component_default(component)
-    File.open(template_path, 'w+') { |file| file.write(contents) }
-  end
-end
+#   if !File.exists?(template_path)
+#     contents = templates.component_default(component)
+#     File.open(template_path, 'w+') { |file| file.write(contents) }
+#   end
+# end
+
+#
+# Create missing .md files
+#
 
 erb_paths =
   Dir.glob("#{ROOT_DIR}/**/*.erb", File::FNM_DOTMATCH).
   to_a.
   filter { |path| !path.start_with?("#{ROOT_DIR}/scripts") }.
   filter { |path| !path.start_with?("#{ROOT_DIR}/distribution/nix") }
 
-#
-# Create missing .md files
-#
 
 erb_paths.each do |erb_path|
   md_path = erb_path.gsub(/\.erb$/, "")

scripts/generate/templates.rb

@@ -43,7 +43,7 @@ def common_component_links(type, limit = 5)
 
     links =
       components.select(&:common?)[0..limit].collect do |component|
-        "[#{component.name}][docs.#{type.to_s.pluralize}.#{component.name}]"
+        "[#{component.name}][#{component_short_link(component)}]"
       end
 
     num_leftover = components.size - links.size
@@ -71,6 +71,14 @@ def component_header(component)
     render("#{partials_path}/_component_header.md", binding).strip
   end
 
+  def component_short_link(component)
+    if component.respond_to?(:sub_type)
+      "docs.#{component.type.pluralize}.#{component.sub_type.pluralize}.#{component.name}"
+    else
+      "docs.#{component.type.pluralize}.#{component.name}"
+    end
+  end
+
   def component_output(component, output, breakout_top_keys: false, heading_depth: 1)
     examples = output.examples
     fields = output.fields ? output.fields.to_h.values.sort : []

scripts/util/metadata/transform.rb

@@ -7,7 +7,8 @@ class Transform < Component
   attr_reader :allow_you_to_description,
     :input_types,
     :output,
-    :output_types
+    :output_types,
+    :sub_type
 
   def initialize(hash)
     super(hash)
@@ -18,6 +19,7 @@ def initialize(hash)
     @input_types = hash.fetch("input_types")
     @output = OpenStruct.new
     @output_types = hash.fetch("output_types")
+    @sub_type = hash.fetch("sub_type")
 
     # checks
 

website/blog/2019-11-25-unit-testing-vector-config-files.md

@@ -18,7 +18,7 @@ mission-critical production pipelines that are collaborated on.
 ## Example
 
 Let's look at a basic example that uses the [`regex_parser`
-transform][docs.transforms.regex_parser] to parse log lines:
+transform][docs.transforms.functions.regex_parser] to parse log lines:
 
 import CodeHeader from '@site/src/components/CodeHeader';
 
@@ -98,5 +98,5 @@ works well for everyone. Please let us know what you think either in our
 [docs.reference.tests]: /docs/reference/tests/
 [docs.sinks.aws_cloudwatch_logs]: /docs/reference/sinks/aws_cloudwatch_logs/
 [docs.sources.file]: /docs/reference/sources/file/
-[docs.transforms.regex_parser]: /docs/reference/transforms/regex_parser/
+[docs.transforms.functions.regex_parser]: /docs/reference/transforms/functions/regex_parser/
 [docs.transforms]: /docs/reference/transforms/

website/blog/2019-12-16-ec2-metadata.md

@@ -7,7 +7,7 @@ tags: ["type: announcement", "domain: transforms", "transform: ec2_metadata"]
 ---
 
 Are your events the laughing-stock of the data warehouse? Then enrich them with
-our brand spanking new [`aws_ec2_metadata` transform][docs.transforms.aws_ec2_metadata].
+our brand spanking new [`aws_ec2_metadata` transform][docs.transforms.functions.aws_ec2_metadata].
 
 <!--truncate-->
 
@@ -29,11 +29,11 @@ don't want all enrichments added then whitelist them with the `fields` option:
   ]
 ```
 
-For more guidance get on the [reference page][docs.transforms.aws_ec2_metadata].
+For more guidance get on the [reference page][docs.transforms.functions.aws_ec2_metadata].
 
 ## Why?
 
 Data is better when it's thicc 👌
 
 
-[docs.transforms.aws_ec2_metadata]: /docs/reference/transforms/aws_ec2_metadata/
+[docs.transforms.functions.aws_ec2_metadata]: /docs/reference/transforms/functions/aws_ec2_metadata/

website/docs/about/concepts.md

@@ -48,7 +48,7 @@ Vector. This might involve parsing, filtering, sampling, or aggregating. You can
 have any number of transforms in your pipeline and how they are composed is up
 to you.
 
-<Jump to="/docs/reference/transforms/">View all transforms</Jump>
+<Jump to="[[[docs.transforms.functions.]]">View all transforms</Jump>
 
 ### Sinks