diff --git a/.gitignore b/.gitignore index 34a9a33..16e68fa 100644 --- a/.gitignore +++ b/.gitignore @@ -134,3 +134,6 @@ type_check/lineprecision.txt demo/issue_41.py demo/issue_80.py setup.sh +/docs/build/doctrees +docs/build/html/.buildinfo +docs/build/html/.buildinfo.bak diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index 9e26bf7..0000000 --- a/.travis.yml +++ /dev/null @@ -1,21 +0,0 @@ -language: python -cache: pip - -matrix: - include: - - python: 3.6 - env: TOXENV=py36 - - python: 3.7 - sudo: true - env: TOXENV=py37 - - python: 3.8 - sudo: true - env: TOXENV=py38 - -install: - - pip install tox - -script: - - tox - -dist: xenial diff --git a/Makefile b/Makefile index a6842b8..acad1b9 100644 --- a/Makefile +++ b/Makefile @@ -21,35 +21,30 @@ build: uv build install-tools: - cd tools && export PATH="/usr/local/go/bin:/usr/local/bin:$PATH" && go mod init mkgherkin && go mod tidy + cd tools && docker pull golang && docker build -t mkgherkin . test: cd features && $(MAKE) all - tox -e py312 + tox run -e py312 test-all: cd features && $(MAKE) all - tox + tox run test-wip: cd features && $(MAKE) all - tox -e wip + tox run -e wip test-tools: - tox -e tools + tox run -e tools cd features && $(MAKE) scan -unit-test: - PYTHONPATH=src python -m pytest -vv --cov=src --cov-report=term-missing ${test} - PYTHONPATH=src python -m doctest tools/*.py - PYTHONPATH=src python -m doctest features/steps/*.py - docs: $(wildcard docs/source/*.rst) PYTHONPATH=src python -m doctest docs/source/*.rst export PYTHONPATH=$(PWD)/src:$(PWD)/tools && cd docs && $(MAKE) html lint: - tox -e lint + tox run -e lint coverage: coverage report -m diff --git a/README.rst b/README.rst index 666a97c..ae44fd5 100644 --- a/README.rst +++ b/README.rst @@ -128,11 +128,11 @@ can then be applied to argument values. >>> ast = env.compile(cel_source) >>> prgm = env.program(ast) - >>> activation = { + >>> context = { ... "account": celpy.json_to_cel({"balance": 500, "overdraftProtection": False}), ... "transaction": celpy.json_to_cel({"withdrawal": 600}) ... } - >>> result = prgm.evaluate(activation) + >>> result = prgm.evaluate(context) >>> result BoolType(False) @@ -153,10 +153,15 @@ https://github.com/google/cel-cpp/blob/master/parser/Cel.g4 https://github.com/google/cel-go/blob/master/parser/gen/CEL.g4 +The documentation includes PlantUML diagrams. +The Sphinx ``conf.py`` provides the location for the PlantUML local JAR file if one is used. +Currently, it expects ``docs/plantuml-asl-1.2025.3.jar``. +The JAR is not provided in this repository, get one from https://plantuml.com. +If you install a different version, update the ``conf.py`` to refer to the JAR file you've downloaded. + Notes ===== - CEL provides a number of runtime errors that are mapped to Python exceptions. - ``no_matching_overload``: this function has no overload for the types of the arguments. @@ -171,6 +176,79 @@ However, see https://github.com/google/cel-spec/blob/master/doc/langdef.md#gradu Rather than try to pre-check types, we'll rely on Python's implementation. +Example 2 +========= + +Here's an example with some details:: + + >>> import celpy + + # A list of type names and class bindings used to create an environment. + >>> types = [] + >>> env = celpy.Environment(types) + + # Parse the code to create the CEL AST. + >>> ast = env.compile("355. / 113.") + + # Use the AST and any overriding functions to create an executable program. + >>> functions = {} + >>> prgm = env.program(ast, functions) + + # Variable bindings. + >>> activation = {} + + # Final evaluation. + >>> try: + ... result = prgm.evaluate(activation) + ... error = None + ... except CELEvalError as ex: + ... result = None + ... error = ex.args[0] + + >>> result # doctest: +ELLIPSIS + DoubleType(3.14159...) + +Example 3 +========= + +See https://github.com/google/cel-go/blob/master/examples/simple_test.go + +The model Go we're sticking close to:: + + d := cel.Declarations(decls.NewVar("name", decls.String)) + env, err := cel.NewEnv(d) + if err != nil { + log.Fatalf("environment creation error: %v\\n", err) + } + ast, iss := env.Compile(`"Hello world! I'm " + name + "."`) + // Check iss for compilation errors. + if iss.Err() != nil { + log.Fatalln(iss.Err()) + } + prg, err := env.Program(ast) + if err != nil { + log.Fatalln(err) + } + out, _, err := prg.Eval(map[string]interface{}{ + "name": "CEL", + }) + if err != nil { + log.Fatalln(err) + } + fmt.Println(out) + // Output:Hello world! I'm CEL. + +Here's the Pythonic approach, using concept patterned after the Go implementation:: + + >>> from celpy import * + >>> decls = {"name": celtypes.StringType} + >>> env = Environment(annotations=decls) + >>> ast = env.compile('"Hello world! I\'m " + name + "."') + >>> out = env.program(ast).evaluate({"name": "CEL"}) + >>> print(out) + Hello world! I'm CEL. + + Contributing ============ diff --git a/benches/complex_expression.py b/benches/complex_expression.py index 7f1fc49..29aa269 100644 --- a/benches/complex_expression.py +++ b/benches/complex_expression.py @@ -351,8 +351,8 @@ "none": lambda optional, : None } -def simple_performance(): - env = celpy.Environment() +def simple_performance(runner_class: type[celpy.Runner] | None = None) -> None: + env = celpy.Environment(runner_class=runner_class) number = 100 compile = timeit.timeit( @@ -364,8 +364,8 @@ def simple_performance(): 'CEL_EXPRESSION_ORIGINAL_NO_OPTIONAL': CEL_EXPRESSION_ORIGINAL_NO_OPTIONAL }, number=number - ) / number - print(f"Compile: {1_000 * compile:9.4f} ms") + ) + print(f"Compile: {1_000 * compile / number:9.4f} ms") ast = env.compile(CEL_EXPRESSION_ORIGINAL_NO_OPTIONAL) @@ -380,8 +380,8 @@ def simple_performance(): 'functions': functions }, number=number - ) / number - print(f"Prepare: {1_000 * prepare:9.4f} ms") + ) + print(f"Prepare: {1_000 * prepare / number:9.4f} ms") program = env.program(ast, functions=functions) @@ -398,8 +398,8 @@ def simple_performance(): """), globals={'celpy': celpy}, number=number - ) / number - print(f"Convert: {1_000 * convert:9.4f} ms") + ) + print(f"Convert: {1_000 * convert / number:9.4f} ms") cel_context = { "class_a": celpy.json_to_cel({"property_a": "something"}), @@ -419,8 +419,8 @@ def simple_performance(): 'cel_context': cel_context }, number=number - ) / number - print(f"Evaluate: {1_000 * evaluation:9.4f} ms") + ) + print(f"Evaluate: {1_000 * evaluation / number:9.4f} ms") print() @@ -450,7 +450,18 @@ def detailed_profile(): ps.print_stats() def main(): - simple_performance() + print("# Performance") + print() + print("## Interpreter") + print() + simple_performance(celpy.InterpretedRunner) + print() + print("## Transpiler") + print() + simple_performance(celpy.CompiledRunner) + print() + print("# Profile") + print() detailed_profile() if __name__ == "__main__": diff --git a/docs/build/doctrees/api.doctree b/docs/build/doctrees/api.doctree index d90706d..9a15a5c 100644 Binary files a/docs/build/doctrees/api.doctree and b/docs/build/doctrees/api.doctree differ diff --git a/docs/build/doctrees/c7n_functions.doctree b/docs/build/doctrees/c7n_functions.doctree index 18c60f8..d159268 100644 Binary files a/docs/build/doctrees/c7n_functions.doctree and b/docs/build/doctrees/c7n_functions.doctree differ diff --git a/docs/build/doctrees/cli.doctree b/docs/build/doctrees/cli.doctree index 2b65e8a..7892a28 100644 Binary files a/docs/build/doctrees/cli.doctree and b/docs/build/doctrees/cli.doctree differ diff --git a/docs/build/doctrees/configuration.doctree b/docs/build/doctrees/configuration.doctree index 83b4843..a82ec52 100644 Binary files a/docs/build/doctrees/configuration.doctree and b/docs/build/doctrees/configuration.doctree differ diff --git a/docs/build/doctrees/environment.pickle b/docs/build/doctrees/environment.pickle index 7600c53..dd0b2a0 100644 Binary files a/docs/build/doctrees/environment.pickle and b/docs/build/doctrees/environment.pickle differ diff --git a/docs/build/doctrees/index.doctree b/docs/build/doctrees/index.doctree index f78755c..0d1d704 100644 Binary files a/docs/build/doctrees/index.doctree and b/docs/build/doctrees/index.doctree differ diff --git a/docs/build/doctrees/installation.doctree b/docs/build/doctrees/installation.doctree index 11aa29a..dd68c27 100644 Binary files a/docs/build/doctrees/installation.doctree and b/docs/build/doctrees/installation.doctree differ diff --git a/docs/build/doctrees/integration.doctree b/docs/build/doctrees/integration.doctree index 4e5e354..de7c8e2 100644 Binary files a/docs/build/doctrees/integration.doctree and b/docs/build/doctrees/integration.doctree differ diff --git a/docs/build/doctrees/structure.doctree b/docs/build/doctrees/structure.doctree index 5c0cba0..ee659a0 100644 Binary files a/docs/build/doctrees/structure.doctree and b/docs/build/doctrees/structure.doctree differ diff --git a/docs/build/html/.buildinfo b/docs/build/html/.buildinfo index 21ac932..88859b4 100644 --- a/docs/build/html/.buildinfo +++ b/docs/build/html/.buildinfo @@ -1,4 +1,4 @@ # Sphinx build info version 1 # This file records the configuration used when building these files. When it is not found, a full rebuild will be done. -config: 17638f2de9b0811238b70ffc830291fb +config: 3f3e9d6f3a661b200fa79b5036f419ce tags: 645f666f9bcd5a90fca523b33c5a78b7 diff --git a/docs/build/html/.buildinfo.bak b/docs/build/html/.buildinfo.bak index ae47034..1791560 100644 --- a/docs/build/html/.buildinfo.bak +++ b/docs/build/html/.buildinfo.bak @@ -1,4 +1,4 @@ # Sphinx build info version 1 # This file records the configuration used when building these files. When it is not found, a full rebuild will be done. -config: 5e7e78792cf4cfb4deae9f0ba2edb8f7 +config: 4e2cb0e805ddcba79237f83ebf4c7647 tags: 645f666f9bcd5a90fca523b33c5a78b7 diff --git a/docs/build/html/_images/plantuml-01f21512c6b94e19efccb4676b518bed63780ddd.png b/docs/build/html/_images/plantuml-01f21512c6b94e19efccb4676b518bed63780ddd.png new file mode 100644 index 0000000..080aa3a Binary files /dev/null and b/docs/build/html/_images/plantuml-01f21512c6b94e19efccb4676b518bed63780ddd.png differ diff --git a/docs/build/html/_images/plantuml-165a23eb11f806bdd308ffac47a78125152ade58.png b/docs/build/html/_images/plantuml-165a23eb11f806bdd308ffac47a78125152ade58.png new file mode 100644 index 0000000..b02b306 Binary files /dev/null and b/docs/build/html/_images/plantuml-165a23eb11f806bdd308ffac47a78125152ade58.png differ diff --git a/docs/build/html/_images/plantuml-59894b154086c2a2ebe8ae46d280279ecce7cf8f.png b/docs/build/html/_images/plantuml-59894b154086c2a2ebe8ae46d280279ecce7cf8f.png new file mode 100644 index 0000000..16f9a9c Binary files /dev/null and b/docs/build/html/_images/plantuml-59894b154086c2a2ebe8ae46d280279ecce7cf8f.png differ diff --git a/docs/build/html/_images/plantuml-60c95188891b5d1267db67bb61a17e9fd7c08060.png b/docs/build/html/_images/plantuml-60c95188891b5d1267db67bb61a17e9fd7c08060.png new file mode 100644 index 0000000..cb501f6 Binary files /dev/null and b/docs/build/html/_images/plantuml-60c95188891b5d1267db67bb61a17e9fd7c08060.png differ diff --git a/docs/build/html/_images/plantuml-a8a77cea7a1f2555dbb1cb6cea10dc21777c76bf.png b/docs/build/html/_images/plantuml-a8a77cea7a1f2555dbb1cb6cea10dc21777c76bf.png new file mode 100644 index 0000000..96c3652 Binary files /dev/null and b/docs/build/html/_images/plantuml-a8a77cea7a1f2555dbb1cb6cea10dc21777c76bf.png differ diff --git a/docs/build/html/_images/plantuml-ac649ac296fc6bea0cee4f6cb2d92533640e6763.png b/docs/build/html/_images/plantuml-ac649ac296fc6bea0cee4f6cb2d92533640e6763.png new file mode 100644 index 0000000..391d74a Binary files /dev/null and b/docs/build/html/_images/plantuml-ac649ac296fc6bea0cee4f6cb2d92533640e6763.png differ diff --git a/docs/build/html/_images/plantuml-c970d97dc7e0a41eb7666fff5d466440fc8d67cf.png b/docs/build/html/_images/plantuml-c970d97dc7e0a41eb7666fff5d466440fc8d67cf.png new file mode 100644 index 0000000..75c2ffd Binary files /dev/null and b/docs/build/html/_images/plantuml-c970d97dc7e0a41eb7666fff5d466440fc8d67cf.png differ diff --git a/docs/build/html/_modules/celpy/__init__.html b/docs/build/html/_modules/celpy/__init__.html index 24e928f..75f87b7 100644 --- a/docs/build/html/_modules/celpy/__init__.html +++ b/docs/build/html/_modules/celpy/__init__.html @@ -47,91 +47,40 @@

Source code for celpy.__init__

 # See the License for the specific language governing permissions and limitations under the License.
 
 """
-Pure Python implementation of CEL.
-
-..  todo:: Consolidate __init__ and parser into one module?
-
-Visible interface to CEL. This exposes the :py:class:`Environment`,
-the :py:class:`Evaluator` run-time, and the :py:mod:`celtypes` module
-with Python types wrapped to be CEL compatible.
-
-Example
-=======
-
-Here's an example with some details::
-
-    >>> import celpy
-
-    # A list of type names and class bindings used to create an environment.
-    >>> types = []
-    >>> env = celpy.Environment(types)
-
-    # Parse the code to create the CEL AST.
-    >>> ast = env.compile("355. / 113.")
-
-    # Use the AST and any overriding functions to create an executable program.
-    >>> functions = {}
-    >>> prgm = env.program(ast, functions)
-
-    # Variable bindings.
-    >>> activation = {}
-
-    # Final evaluation.
-    >>> try:
-    ...    result = prgm.evaluate(activation)
-    ...    error = None
-    ... except CELEvalError as ex:
-    ...    result = None
-    ...    error = ex.args[0]
-
-    >>> result  # doctest: +ELLIPSIS
-    DoubleType(3.14159...)
-
-Another Example
-===============
-
-See https://github.com/google/cel-go/blob/master/examples/simple_test.go
-
-The model Go we're sticking close to::
-
-    d := cel.Declarations(decls.NewVar("name", decls.String))
-    env, err := cel.NewEnv(d)
-    if err != nil {
-        log.Fatalf("environment creation error: %v\\n", err)
-    }
-    ast, iss := env.Compile(`"Hello world! I'm " + name + "."`)
-    // Check iss for compilation errors.
-    if iss.Err() != nil {
-        log.Fatalln(iss.Err())
-    }
-    prg, err := env.Program(ast)
-    if err != nil {
-        log.Fatalln(err)
-    }
-    out, _, err := prg.Eval(map[string]interface{}{
-        "name": "CEL",
-    })
-    if err != nil {
-        log.Fatalln(err)
-    }
-    fmt.Println(out)
-    // Output:Hello world! I'm CEL.
-
-Here's the Pythonic approach, using concept patterned after the Go implementation::
-
-    >>> from celpy import *
-    >>> decls = {"name": celtypes.StringType}
-    >>> env = Environment(annotations=decls)
-    >>> ast = env.compile('"Hello world! I\\'m " + name + "."')
-    >>> out = env.program(ast).evaluate({"name": "CEL"})
-    >>> print(out)
-    Hello world! I'm CEL.
+The pure Python implementation of the Common Expression Language, CEL.
 
+This module defines an interface to CEL for integration into other Python applications.
+This exposes the :py:class:`Environment` used to compile the source module,
+the :py:class:`Runner` used to evaluate the compiled code,
+and the :py:mod:`celpy.celtypes` module with Python types wrapped to be CEL compatible.
+
+The way these classes are used is as follows:
+
+..  uml::
+
+    @startuml
+    start
+    :Gather (or define) annotations;
+    :Create ""Environment"";
+    :Compile CEL;
+    :Create ""Runner"" with any extension functions;
+    :Evaluate the ""Runner"" with a ""Context"";
+    stop
+    @enduml
+
+The explicit decomposition into steps permits
+two extensions:
+
+1.  Transforming the AST to introduce any optimizations.
+
+2.  Saving the :py:class:`Runner` instance to reuse an expression with new inputs.
 """
 
+import abc
 import json  # noqa: F401
 import logging
 import sys
+from textwrap import indent
 from typing import Any, Dict, Optional, Type, cast
 
 import lark
@@ -151,6 +100,8 @@ 

Source code for celpy.__init__

     Context,
     Evaluator,
     Result,
+    Transpiler,
+    TranspilerTree,
     base_functions,
 )
 
@@ -160,15 +111,28 @@ 

Source code for celpy.__init__

 
 
[docs] -class Runner: - """Abstract runner. +class Runner(abc.ABC): + """Abstract runner for a compiled CEL program. - Given an AST, this can evaluate the AST in the context of a specific activation - with any override function definitions. + The :py:class:`Environment` creates a :py:class:`Runner` to permit + saving a ready-tp-evaluate, compiled CEL expression. + A :py:class:`Runner` will evaluate the AST in the context of a specific activation + with the provided variable values. - .. todo:: add type adapter and type provider registries. + A :py:class:`Runner` class provides + the ``tree_node_class`` attribute to define the type for tree nodes. + This class information is used by the :py:class:`Environment` to tailor the ``Lark`` instance created. + The class named by the ``tree_node_class`` can include specialized AST features + needed by a :py:class:`Runner` instance. + + .. todo:: For a better fit with Go language expectations + + Consider addoing type adapter and type provider registries. + This would permit distinct sources of protobuf message types. """ + tree_node_class: type = lark.Tree +
[docs] def __init__( @@ -177,25 +141,50 @@

Source code for celpy.__init__

         ast: lark.Tree,
         functions: Optional[Dict[str, CELFunction]] = None,
     ) -> None:
+        """
+        Initialize this ``Runner`` with a given AST.
+        The Runner will have annotations take from the :py:class:`Environment`,
+        plus any unique functions defined here.
+        """
         self.logger = logging.getLogger(f"celpy.{self.__class__.__name__}")
         self.environment = environment
         self.ast = ast
         self.functions = functions
+
+[docs] + def __repr__(self) -> str: + return f"{self.__class__.__name__}({self.environment}, {self.ast}, {self.functions})"
+ +
[docs] - def new_activation(self, context: Context) -> Activation: + def new_activation(self) -> Activation: """ - Builds the working activation from the environmental defaults. + Builds a new, working :py:class:`Activation` using the :py:class:`Environment` as defaults. + A Context will later be layered onto this for evaluation. """ - return self.environment.activation().nested_activation(vars=context)
+ base_activation = Activation( + package=self.environment.package, + annotations=self.environment.annotations, + functions=self.functions, + ) + return base_activation
[docs] + @abc.abstractmethod def evaluate(self, activation: Context) -> celpy.celtypes.Value: # pragma: no cover - raise NotImplementedError
+ """ + Given variable definitions in the :py:class:`celpy.evaluation.Context`, evaluate the given AST and return the resulting value. + + Generally, this should raise an :exc:`CELEvalError` for most kinds of ordinary problems. + It may raise an :exc:`CELUnsupportedError` for future features that aren't fully implemented. + Any Python exception reflects a serious problem. + """ + ...
@@ -204,16 +193,7 @@

Source code for celpy.__init__

 [docs]
 class InterpretedRunner(Runner):
     """
-    Pure AST expression evaluator. Uses :py:class:`evaluation.Evaluator` class.
-
-    Given an AST, this evauates the AST in the context of a specific activation.
-
-    The returned value will be a celtypes type.
-
-    Generally, this should raise an :exc:`CELEvalError` for most kinds of ordinary problems.
-    It may raise an :exc:`CELUnsupportedError` for future features.
-
-    ..  todo:: Refractor the Evaluator constructor from evaluation.
+    An **Adapter** for the :py:class:`celpy.evaluation.Evaluator` class.
     """
 
 
@@ -221,10 +201,9 @@

Source code for celpy.__init__

     def evaluate(self, context: Context) -> celpy.celtypes.Value:
         e = Evaluator(
             ast=self.ast,
-            activation=self.new_activation(context),
-            functions=self.functions,
+            activation=self.new_activation(),
         )
-        value = e.evaluate()
+        value = e.evaluate(context)
         return value
@@ -234,42 +213,57 @@

Source code for celpy.__init__

 [docs]
 class CompiledRunner(Runner):
     """
-    Python compiled expression evaluator. Uses Python byte code and :py:func:`eval`.
+    An **Adapter** for the :py:class:`celpy.evaluation.Transpiler` class.
 
-    Given an AST, this evaluates the AST in the context of a specific activation.
+    A :py:class:`celpy.evaluation.Transpiler` instance transforms the AST into Python.
+    It uses :py:func:`compile` to create a code object.
+    The final :py:meth:`evaluate` method uses  :py:func:`exec` to evaluate the code object.
 
-    Transform the AST into Python, uses :py:func:`compile` to create a code object.
-    Uses :py:func:`eval` to evaluate.
+    Note, this requires the ``celpy.evaluation.TranspilerTree`` classes
+    instead of the default ``lark.Tree`` class.
     """
 
+    tree_node_class: type = TranspilerTree
+
 
[docs] def __init__( self, environment: "Environment", - ast: lark.Tree, + ast: TranspilerTree, functions: Optional[Dict[str, CELFunction]] = None, ) -> None: - super().__init__(environment, ast, functions)
+ """ + Transpile to Python, and use :py:func:`compile` to create a code object. + """ + super().__init__(environment, ast, functions) + self.tp = Transpiler( + ast=cast(TranspilerTree, self.ast), + activation=self.new_activation(), + ) + self.tp.transpile() + self.logger.info("Transpiled:\n%s", indent(self.tp.source_text, " "))
- # Transform AST to Python. - # compile() - # cache executable code object.
[docs] - def evaluate(self, activation: Context) -> celpy.celtypes.Value: - # eval() code object with activation as locals, and built-ins as gobals. - return super().evaluate(activation)
+ def evaluate(self, context: Context) -> celpy.celtypes.Value: + """ + Use :py:func:`exec` to execute the code object. + """ + value = self.tp.evaluate(context) + return value
-# TODO: Refactor classes into a separate "cel_protobuf" module. -# TODO: Becomes cel_protobuf.Int32Value +# TODO: Refactor this class into a separate "cel_protobuf" module. +# TODO: Rename this type to ``cel_protobuf.Int32Value``
[docs] class Int32Value(celpy.celtypes.IntType): + """A wrapper for int32 values.""" +
[docs] def __new__( @@ -287,8 +281,8 @@

Source code for celpy.__init__

 
 
 
-# The "well-known" types in a google.protobuf package.
-# We map these to CEl types instead of defining additional Protobuf Types.
+# The "well-known" types in a ``google.protobuf`` package.
+# We map these to CEL types instead of defining additional Protobuf Types.
 # This approach bypasses some of the range constraints that are part of these types.
 # It may also cause values to compare as equal when they were originally distinct types.
 googleapis = {
@@ -309,15 +303,19 @@ 

Source code for celpy.__init__

 
[docs] class Environment: - """Compiles CEL text to create an Expression object. + """ + Contains the current evaluation context. + The :py:meth:`Environment.compile` method + compiles CEL text to create an AST. - From the Go implementation, there are things to work with the type annotations: + The :py:meth:`Environment.program` method + packages the AST into a program ready for evaluation. - - type adapters registry make other native types available for CEL. + .. todo:: For a better fit with Go language expectations - - type providers registry make ProtoBuf types available for CEL. + - A type adapters registry makes other native types available for CEL. - .. todo:: Add adapter and provider registries to the Environment. + - A type providers registry make ProtoBuf types available for CEL. """
@@ -348,7 +346,7 @@

Source code for celpy.__init__

         self.annotations: Dict[str, Annotation] = annotations or {}
         self.logger.debug("Type Annotations %r", self.annotations)
         self.runner_class: Type[Runner] = runner_class or InterpretedRunner
-        self.cel_parser = CELParser()
+        self.cel_parser = CELParser(tree_class=self.runner_class.tree_node_class)
         self.runnable: Runner
 
         # Fold in standard annotations. These (generally) define well-known protobuf types.
@@ -357,6 +355,12 @@ 

Source code for celpy.__init__

         # We'd like to add 'type.googleapis.com/google' directly, but it seems to be an alias
         # for 'google', the path after the '/' in the uri.
 
+
+[docs] + def __repr__(self) -> str: + return f"{self.__class__.__name__}({self.package}, {self.annotations}, {self.runner_class})"
+ +
[docs] def compile(self, text: str) -> Expression: @@ -370,19 +374,18 @@

Source code for celpy.__init__

     def program(
         self, expr: lark.Tree, functions: Optional[Dict[str, CELFunction]] = None
     ) -> Runner:
-        """Transforms the AST into an executable runner."""
+        """
+        Transforms the AST into an executable :py:class:`Runner` object.
+
+        :param expr: The parse tree from :py:meth:`compile`.
+        :param functions: Any additional functions to be used by this CEL expression.
+        :returns: A :py:class:`Runner` instance that can be evaluated with a ``Context`` that provides values.
+        """
         self.logger.debug("Package %r", self.package)
         runner_class = self.runner_class
         self.runnable = runner_class(self, expr, functions)
+        self.logger.debug("Runnable %r", self.runnable)
         return self.runnable
- - -
-[docs] - def activation(self) -> Activation: - """Returns a base activation""" - activation = Activation(package=self.package, annotations=self.annotations) - return activation
@@ -412,14 +415,15 @@

CEL in Python

Navigation

-

Contents:

+

Documentation Content:

diff --git a/docs/build/html/_modules/celpy/__main__.html b/docs/build/html/_modules/celpy/__main__.html index 56c7530..dc18257 100644 --- a/docs/build/html/_modules/celpy/__main__.html +++ b/docs/build/html/_modules/celpy/__main__.html @@ -47,85 +47,25 @@

Source code for celpy.__main__

 # See the License for the specific language governing permissions and limitations under the License.
 
 """
-Pure Python implementation of CEL.
-
-This provides a few jq-like, bc-like, and shell expr-like features.
-
--   ``jq`` uses ``.`` to refer the current document. By setting a package
-    name of ``"jq"`` and placing the JSON object in the package, we achieve
-    similar syntax.
-
--   ``bc`` offers complex function definitions and other programming support.
-    CEL can only evaluate a few bc-like expressions.
-
--   This does everything ``expr`` does, but the syntax is slightly different.
-    The output of comparisons -- by default -- is boolean, where ``expr`` is an integer 1 or 0.
-    Use ``-f 'd'`` to see decimal output instead of Boolean text values.
-
--   This does some of what ``test`` does, without a lot of the sophisticated
-    file system data gathering.
-    Use ``-b`` to set the exit status code from a Boolean result.
-
-TODO: This can also have a REPL, as well as process CSV files.
-
-SYNOPSIS
-========
-
-::
-
-    python -m celpy [--arg name:type=value ...] [--null-input] expr
-
-Options:
-
-:--arg:
-    Provides argument names, types and optional values.
-    If the value is not provided, the name is expected to be an environment
-    variable, and the value of the environment variable is converted and used.
-
-:--null-input:
-    Normally, JSON documents are read from stdin in ndjson format. If no JSON documents are
-    provided, the ``--null-input`` option skips trying to read from stdin.
-
-:expr:
-    A CEL expression to evaluate.
-
-JSON documents are read from stdin in NDJSON format (http://jsonlines.org/, http://ndjson.org/).
-For each JSON document, the expression is evaluated with the document in a default
-package. This allows `.name` to pick items from the document.
-
-By default, the output is JSON serialized. This means strings will be JSON-ified and have quotes.
-
-If a ``--format`` option is provided, this is applied to the resulting object; this can be
-used to strip quotes, or limit precision on double objects, or convert numbers to hexadecimal.
-
-Arguments, Types, and Namespaces
-================================
-
-CEL objects rely on the celtypes definitions.
-
-Because of the close association between CEL and protobuf, some well-known protobuf types
-are also supported.
-
-..  todo:: CLI type environment
-
-    Permit name.name:type=value to create namespace bindings.
-
-Further, type providers can be bound to CEL. This means an extended CEL
-may have additional types beyond those defined by the :py:class:`Activation` class.
+The CLI interface to ``celpy``.
 
+This parses the command-line options.
+It also offers an interactive REPL.
 """
 
 import argparse
 import ast
 import cmd
+import datetime
 import json
 import logging
 import logging.config
 import os
 from pathlib import Path
 import re
+import stat as os_stat
 import sys
-from typing import Any, Callable, Dict, List, Optional, Tuple, cast
+from typing import Any, Callable, Dict, List, Optional, Tuple, Union, cast
 
 try:
     import tomllib
@@ -223,7 +163,7 @@ 

Source code for celpy.__main__

             type_definition = CLI_ARG_TYPES[type_name]
             value = cast(
                 celtypes.Value,
-                type_definition(value_text),  # type: ignore[arg-type, call-arg]
+                type_definition(value_text),  # type: ignore [call-arg]
             )
         except KeyError:
             raise argparse.ArgumentTypeError(
@@ -334,6 +274,76 @@ 

Source code for celpy.__main__

 
 
 
+
+[docs] +def stat(path: Union[Path, str]) -> Optional[celtypes.MapType]: + """This function is added to the CLI to permit file-system interrogation.""" + try: + status = Path(path).stat() + data = { + "st_atime": celtypes.TimestampType( + datetime.datetime.fromtimestamp(status.st_atime) + ), + "st_birthtime": celtypes.TimestampType( + datetime.datetime.fromtimestamp(status.st_birthtime) + ), + "st_ctime": celtypes.TimestampType( + datetime.datetime.fromtimestamp(status.st_ctime) + ), + "st_dev": celtypes.IntType(status.st_dev), + "st_ino": celtypes.IntType(status.st_ino), + "st_mtime": celtypes.TimestampType( + datetime.datetime.fromtimestamp(status.st_mtime) + ), + "st_nlink": celtypes.IntType(status.st_nlink), + "st_size": celtypes.IntType(status.st_size), + "group_access": celtypes.BoolType(status.st_gid == os.getegid()), + "user_access": celtypes.BoolType(status.st_uid == os.geteuid()), + } + + # From mode File type: + # - block, character, directory, regular, symbolic link, named pipe, socket + # One predicate should be True; we want the code for that key. + data["kind"] = celtypes.StringType( + { + predicate(status.st_mode): code + for code, predicate in [ + ("b", os_stat.S_ISBLK), + ("c", os_stat.S_ISCHR), + ("d", os_stat.S_ISDIR), + ("f", os_stat.S_ISREG), + ("p", os_stat.S_ISFIFO), + ("l", os_stat.S_ISLNK), + ("s", os_stat.S_ISSOCK), + ] + }.get(True, "?") + ) + + # Special bits: uid, gid, sticky + data["setuid"] = celtypes.BoolType((os_stat.S_ISUID & status.st_mode) != 0) + data["setgid"] = celtypes.BoolType((os_stat.S_ISGID & status.st_mode) != 0) + data["sticky"] = celtypes.BoolType((os_stat.S_ISVTX & status.st_mode) != 0) + + # permissions, limited to user-level RWX, nothing more. + data["r"] = celtypes.BoolType(os.access(path, os.R_OK)) + data["w"] = celtypes.BoolType(os.access(path, os.W_OK)) + data["x"] = celtypes.BoolType(os.access(path, os.X_OK)) + try: + extra = { + "st_blksize": celtypes.IntType(status.st_blksize), + "st_blocks": celtypes.IntType(status.st_blocks), + "st_flags": celtypes.IntType(status.st_flags), + "st_rdev": celtypes.IntType(status.st_rdev), + "st_gen": celtypes.IntType(status.st_gen), + } + except NameError: # pragma: no cover + extra = {} + return celtypes.MapType(data | extra) + except FileNotFoundError: + return None
+ + +
[docs] class CEL_REPL(cmd.Cmd): @@ -397,6 +407,7 @@

Source code for celpy.__main__

 
     do_exit = do_quit
     do_bye = do_quit
+    do_EOF = do_quit
 
 
[docs] @@ -431,10 +442,10 @@

Source code for celpy.__main__

     """
     try:
         activation[variable] = json.loads(document, cls=CELJSONDecoder)
-        result = prgm.evaluate(activation)
-        display(result)
-        if boolean_to_status and isinstance(result, (celtypes.BoolType, bool)):
-            return 0 if result else 1
+        result_value = prgm.evaluate(activation)
+        display(result_value)
+        if boolean_to_status and isinstance(result_value, (celtypes.BoolType, bool)):
+            return 0 if result_value else 1
         return 0
     except CELEvalError as ex:
         # ``jq`` KeyError problems result in ``None``.
@@ -481,12 +492,12 @@ 

Source code for celpy.__main__

 
     if options.format:
 
-        def output_display(result: Result) -> None:
-            print("{0:{format}}".format(result, format=options.format))
+        def output_display(result_value: Result) -> None:
+            print("{0:{format}}".format(result_value, format=options.format))
     else:
 
-        def output_display(result: Result) -> None:
-            print(json.dumps(result, cls=CELJSONEncoder))
+        def output_display(result_value: Result) -> None:
+            print(json.dumps(result_value, cls=CELJSONEncoder))
 
     logger.info("Expr: %r", options.expr)
 
@@ -497,17 +508,18 @@ 

Source code for celpy.__main__

     if options.arg:
         annotations = {name: type for name, type, value in options.arg}
     else:
-        annotations = None
+        annotations = {}
+    annotations["stat"] = celtypes.FunctionType
 
     # If we're creating a named JSON document, we don't provide a default package.
-    # If we're usinga  JSON document to populate a package, we provide the given name.
+    # If we're using a  JSON document to populate a package, we provide the given name.
     env = Environment(
         package=None if options.null_input else options.package,
         annotations=annotations,
     )
     try:
         expr = env.compile(options.expr)
-        prgm = env.program(expr)
+        prgm = env.program(expr, functions={"stat": stat})
     except CELParseError as ex:
         print(
             env.cel_parser.error_text(ex.args[0], ex.line, ex.column), file=sys.stderr
@@ -522,17 +534,19 @@ 

Source code for celpy.__main__

     if options.null_input:
         # Don't read stdin, evaluate with only the activation context.
         try:
-            result = prgm.evaluate(activation)
+            result_value = prgm.evaluate(activation)
             if options.boolean:
-                if isinstance(result, (celtypes.BoolType, bool)):
-                    summary = 0 if result else 1
+                if isinstance(result_value, (celtypes.BoolType, bool)):
+                    summary = 0 if result_value else 1
                 else:
                     logger.warning(
-                        "Expected celtypes.BoolType, got %s = %r", type(result), result
+                        "Expected celtypes.BoolType, got %s = %r",
+                        type(result_value),
+                        result_value,
                     )
                     summary = 2
             else:
-                output_display(result)
+                output_display(result_value)
                 summary = 0
         except CELEvalError as ex:
             print(
@@ -629,14 +643,15 @@ 

CEL in Python

Navigation

-

Contents:

+

Documentation Content:

diff --git a/docs/build/html/_modules/celpy/adapter.html b/docs/build/html/_modules/celpy/adapter.html new file mode 100644 index 0000000..df103ae --- /dev/null +++ b/docs/build/html/_modules/celpy/adapter.html @@ -0,0 +1,269 @@ + + + + + + + celpy.adapter — CEL in Python documentation + + + + + + + + + + + + + + + + + + +
+
+
+ + +
+ +

Source code for celpy.adapter

+# SPDX-Copyright: Copyright (c) Capital One Services, LLC
+# SPDX-License-Identifier: Apache-2.0
+# Copyright 2020 Capital One Services, LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and limitations under the License.
+
+"""
+Converts some Python-native types into CEL structures.
+
+Currently, atomic Python objects have direct use of types in :mod:`celpy.celtypes`.
+
+Non-atomic Python objects are characterized by JSON and Protobuf messages.
+This module has functions to convert JSON objects to CEL.
+
+A proper protobuf decoder is TBD.
+
+A more sophisticated type injection capability may be needed to permit
+additional types or extensions to :mod:`celpy.celtypes`.
+"""
+
+import base64
+import datetime
+import json
+from typing import Any, Dict, List, Union, cast
+
+from celpy import celtypes
+
+JSON = Union[Dict[str, Any], List[Any], bool, float, int, str, None]
+
+
+
+[docs] +class CELJSONEncoder(json.JSONEncoder): + """ + An Encoder to export CEL objects as JSON text. + + This is **not** a reversible transformation. Some things are coerced to strings + without any more detailed type marker. + Specifically timestamps, durations, and bytes. + """ + +
+[docs] + @staticmethod + def to_python( + cel_object: celtypes.Value, + ) -> Union[celtypes.Value, List[Any], Dict[Any, Any], bool]: + """Recursive walk through the CEL object, replacing BoolType with native bool instances. + This lets the :py:mod:`json` module correctly represent the obects + with JSON ``true`` and ``false``. + + This will also replace ListType and MapType with native ``list`` and ``dict``. + All other CEL objects will be left intact. This creates an intermediate hybrid + beast that's not quite a :py:class:`celtypes.Value` because a few things have been replaced. + """ + if isinstance(cel_object, celtypes.BoolType): + return True if cel_object else False + elif isinstance(cel_object, celtypes.ListType): + return [CELJSONEncoder.to_python(item) for item in cel_object] + elif isinstance(cel_object, celtypes.MapType): + return { + CELJSONEncoder.to_python(key): CELJSONEncoder.to_python(value) + for key, value in cel_object.items() + } + else: + return cel_object
+ + +
+[docs] + def encode(self, cel_object: celtypes.Value) -> str: + """ + Override built-in encode to create proper Python :py:class:`bool` objects. + """ + return super().encode(CELJSONEncoder.to_python(cel_object))
+ + +
+[docs] + def default(self, cel_object: celtypes.Value) -> JSON: + if isinstance(cel_object, celtypes.TimestampType): + return str(cel_object) + elif isinstance(cel_object, celtypes.DurationType): + return str(cel_object) + elif isinstance(cel_object, celtypes.BytesType): + return base64.b64encode(cel_object).decode("ASCII") + else: + return cast(JSON, super().default(cel_object))
+
+ + + +
+[docs] +class CELJSONDecoder(json.JSONDecoder): + """ + An Encoder to import CEL objects from JSON to the extent possible. + + This does not handle non-JSON types in any form. Coercion from string + to TimestampType or DurationType or BytesType is handled by celtype + constructors. + """ + +
+[docs] + def decode(self, source: str, _w: Any = None) -> Any: + raw_json = super().decode(source) + return json_to_cel(raw_json)
+
+ + + +
+[docs] +def json_to_cel(document: JSON) -> celtypes.Value: + """Convert parsed JSON object from Python to CEL to the extent possible. + + It's difficult to distinguish strings which should be timestamps or durations. + + :: + + >>> from pprint import pprint + >>> from celpy.adapter import json_to_cel + >>> doc = json.loads('["str", 42, 3.14, null, true, {"hello": "world"}]') + >>> cel = json_to_cel(doc) + >>> pprint(cel) + ListType([StringType('str'), IntType(42), DoubleType(3.14), None, BoolType(True), \ +MapType({StringType('hello'): StringType('world')})]) + """ + if isinstance(document, bool): + return celtypes.BoolType(document) + elif isinstance(document, float): + return celtypes.DoubleType(document) + elif isinstance(document, int): + return celtypes.IntType(document) + elif isinstance(document, str): + return celtypes.StringType(document) + elif document is None: + return None + elif isinstance(document, (tuple, List)): + return celtypes.ListType([json_to_cel(item) for item in document]) + elif isinstance(document, Dict): + return celtypes.MapType( + {json_to_cel(key): json_to_cel(value) for key, value in document.items()} + ) + elif isinstance(document, datetime.datetime): + return celtypes.TimestampType(document) + elif isinstance(document, datetime.timedelta): + return celtypes.DurationType(document) + else: + raise ValueError( + f"unexpected type {type(document)} in JSON structure {document!r}" + )
+ +
+ +
+ +
+
+ +
+
+ + + + + + + \ No newline at end of file diff --git a/docs/build/html/_modules/celpy/c7nlib.html b/docs/build/html/_modules/celpy/c7nlib.html new file mode 100644 index 0000000..c18d332 --- /dev/null +++ b/docs/build/html/_modules/celpy/c7nlib.html @@ -0,0 +1,1988 @@ + + + + + + + celpy.c7nlib — CEL in Python documentation + + + + + + + + + + + + + + + + + + +
+
+
+ + +
+ +

Source code for celpy.c7nlib

+# SPDX-Copyright: Copyright (c) Capital One Services, LLC
+# SPDX-License-Identifier: Apache-2.0
+# Copyright 2020 Capital One Services, LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and limitations under the License.
+
+"""
+Functions for C7N features when evaluating CEL expressions.
+
+These functions provide a mapping between C7N features and CEL.
+
+These functions are exposed by the global ``FUNCTIONS`` dictionary that is provided
+to the CEL evaluation run-time to provide necessary C7N features.
+
+The functions rely on implementation details in the ``CELFilter`` class.
+
+The API
+=======
+
+A C7N implementation can use CEL expressions and the :py:mod:`c7nlib` module as follows::
+
+    class CELFilter(c7n.filters.core.Filter):
+        decls = {
+            "resource": celpy.celtypes.MapType,
+            "now": celpy.celtypes.TimestampType,
+            "event": celpy.celtypes.MapType,
+        }
+        decls.update(celpy.c7nlib.DECLARATIONS)
+
+        def __init__(self, expr: str) -> None:
+            self.expr = expr
+
+        def validate(self) -> None:
+            cel_env = celpy.Environment(
+                annotations=self.decls,
+                runner_class=c7nlib.C7N_Interpreted_Runner)
+            cel_ast = cel_env.compile(self.expr)
+            self.pgm = cel_env.program(cel_ast, functions=celpy.c7nlib.FUNCTIONS)
+
+        def process(self,
+            resources: Iterable[celpy.celtypes.MapType]) -> Iterator[celpy.celtypes.MapType]:
+            now = datetime.datetime.utcnow()
+            for resource in resources:
+                with C7NContext(filter=the_filter):
+                    cel_activation = {
+                        "resource": celpy.json_to_cel(resource),
+                        "now": celpy.celtypes.TimestampType(now),
+                    }
+                    if self.pgm.evaluate(cel_activation):
+                        yield resource
+
+The :py:mod:`celpy.c7nlib` library of functions is bound into the CEL :py:class:`celpy.__init__.Runner` object  that's built from the AST.
+
+Several variables will be required in the :py:class:`celpy.evaluation.Activation` for use by most CEL expressions
+that implement C7N filters:
+
+:resource:
+    A JSON document describing a cloud resource.
+
+:now:
+    The current timestamp.
+
+:event:
+    May be needed; it should  be a JSON document describing an AWS CloudWatch event.
+
+
+The type: value Features
+========================
+
+The core value features of C7N require a number of CEL extensions.
+
+-   :func:`glob(string, pattern)` uses Python fnmatch rules. This implements ``op: glob``.
+
+-   :func:`difference(list, list)` creates intermediate sets and computes the difference
+    as a boolean value. Any difference is True.  This implements ``op: difference``.
+
+-   :func:`intersect(list, list)` creats intermediate sets and computes the intersection
+    as a boolean value. Any interection is True.  This implements ``op: intersect``.
+
+-   :func:`normalize(string)` supports normalized comparison between strings.
+    In this case, it means lower cased and trimmed. This implements ``value_type: normalize``.
+
+-   :func:`net.cidr_contains` checks to see if a given CIDR block contains a specific
+    address.  See https://www.openpolicyagent.org/docs/latest/policy-reference/#net.
+
+-   :func:`net.cidr_size` extracts the prefix length of a parsed CIDR block.
+
+-   :func:`version` uses ``disutils.version.LooseVersion`` to compare version strings.
+
+-   :func:`resource_count` function. This is TBD.
+
+The type: value_from features
+==============================
+
+This relies on  ``value_from()`` and ``jmes_path_map()`` functions
+
+In context, it looks like this::
+
+    value_from("s3://c7n-resources/exemptions.json", "json")
+    .jmes_path_map('exemptions.ec2.rehydration.["IamInstanceProfile.Arn"][].*[].*[]')
+    .contains(resource["IamInstanceProfile"]["Arn"])
+
+The ``value_from()`` function reads values from a given URI.
+
+-   A full URI for an S3 bucket.
+
+-   A full URI for a server that supports HTTPS GET requests.
+
+If a format is given, this is used, otherwise it's based on the
+suffix of the path.
+
+The ``jmes_path_map()`` function compiles and applies a JMESPath
+expression against each item in the collection to create a
+new collection.  To an extent, this repeats functionality
+from the ``map()`` macro.
+
+Additional Functions
+====================
+
+A number of C7N subclasses of ``Filter`` provide additional features. There are
+at least 70-odd functions that are expressed or implied by these filters.
+
+Because the CEL expressions are always part of a ``CELFilter``, all of these
+additional C7N features need to be transformed into "mixins" that are implemented
+in two places. The function is part of the legacy subclass of ``Filter``,
+and the function is also part of ``CELFilter``.
+
+::
+
+    class InstanceImageMixin:
+        # from :py:class:`InstanceImageBase` refactoring
+        def get_instance_image(self):
+            pass
+
+    class RelatedResourceMixin:
+        # from :py:class:`RelatedResourceFilter` mixin
+        def get_related_ids(self):
+            pass
+
+        def get_related(self):
+            pass
+
+    class CredentialReportMixin:
+        # from :py:class:`c7n.resources.iam.CredentialReport` filter.
+        def get_credential_report(self):
+            pass
+
+    class ResourceKmsKeyAliasMixin:
+        # from :py:class:`c7n.resources.kms.ResourceKmsKeyAlias`
+        def get_matching_aliases(self, resource):
+            pass
+
+    class CrossAccountAccessMixin:
+        # from :py:class:`c7n.filters.iamaccessfilter.CrossAccountAccessFilter`
+        def get_accounts(self, resource):
+            pass
+        def get_vpcs(self, resource):
+            pass
+        def get_vpces(self, resource):
+            pass
+        def get_orgids(self, resource):
+            pass
+        # from :py:class:`c7n.resources.secretsmanager.CrossAccountAccessFilter`
+        def get_resource_policy(self, resource):
+            pass
+
+    class SNSCrossAccountMixin:
+        # from :py:class:`c7n.resources.sns.SNSCrossAccount`
+        def get_endpoints(self, resource):
+            pass
+        def get_protocols(self, resource):
+            pass
+
+    class ImagesUnusedMixin:
+        # from :py:class:`c7n.resources.ami.ImageUnusedFilter`
+        def _pull_ec2_images(self, resource):
+            pass
+        def _pull_asg_images(self, resource):
+            pass
+
+    class SnapshotUnusedMixin:
+        # from :py:class:`c7n.resources.ebs.SnapshotUnusedFilter`
+        def _pull_asg_snapshots(self, resource):
+            pass
+        def _pull_ami_snapshots(self, resource):
+            pass
+
+    class IamRoleUsageMixin:
+        # from :py:class:`c7n.resources.iam.IamRoleUsage`
+        def service_role_usage(self, resource):
+            pass
+        def instance_profile_usage(self, resource):
+            pass
+
+    class SGUsageMixin:
+        # from :py:class:`c7n.resources.vpc.SGUsage`
+        def scan_groups(self, resource):
+            pass
+
+    class IsShieldProtectedMixin:
+        # from :py:mod:`c7n.resources.shield`
+        def get_type_protections(self, resource):
+            pass
+
+    class ShieldEnabledMixin:
+        # from :py:class:`c7n.resources.account.ShieldEnabled`
+        def account_shield_subscriptions(self, resource):
+            pass
+
+    class CELFilter(
+        InstanceImageMixin, RelatedResourceMixin, CredentialReportMixin,
+        ResourceKmsKeyAliasMixin, CrossAccountAccessMixin, SNSCrossAccountMixin,
+        ImagesUnusedMixin, SnapshotUnusedMixin, IamRoleUsageMixin, SGUsageMixin,
+        Filter,
+    ):
+        '''Container for functions used by c7nlib to expose data to CEL'''
+        def __init__(self, data, manager) -> None:
+            super().__init__(data, manager)
+            assert data["type"].lower() == "cel"
+            self.expr = data["expr"]
+            self.parser = c7n.filters.offhours.ScheduleParser()
+
+        def validate(self):
+            pass  # See above example
+
+        def process(self, resources):
+            pass  # See above example
+
+This is not the complete list. See the ``tests/test_c7nlib.py`` for the ``celfilter_instance``
+fixture which contains **all** of the functions required.
+
+C7N Context Object
+==================
+
+A number of the functions require access to C7N features that are not simply part
+of the resource being filtered. There are two alternative ways to handle this dependency:
+
+-   A global C7N context object that has the current ``CELFilter`` providing
+    access to C7N internals.
+
+-   A ``C7N`` argument to the functions that need C7N access.
+    This would be provided in the activation context for CEL.
+
+To keep the library functions looking simple, the module global ``C7N`` is used.
+This avoids introducing a non-CEL parameter to the :py:mod:`celpy.c7nlib` functions.
+
+The ``C7N`` context object contains the following attributes:
+
+:filter:
+    The original C7N ``Filter`` object. This provides access to the
+    resource manager. It can be used to manage supplemental
+    queries using C7N caches and other resource management.
+
+This is set by the :py:class:`C7NContext` prior to CEL evaluation.
+
+Name Resolution
+===============
+
+Note that names are **not** resolved via a lookup in the program object,
+an instance of the :py:class:`celpy.Runner` class. To keep these functions
+simple, the runner is not part of the run-time, and name resolution
+will appear to be "hard-wrired" among these functions.
+
+This is rarely an issue, since most of these functions are independent.
+The :func:`value_from` function relies on :func:`text_from` and :func:`parse_text`.
+Changing either of these functions with an override won't modify the behavior
+of :func:`value_from`.
+"""
+
+import csv
+import fnmatch
+import io
+import ipaddress
+import json
+import logging
+import os.path
+import sys
+import urllib.request
+import zlib
+from contextlib import closing
+from packaging.version import Version
+from types import TracebackType
+from typing import Any, Callable, Dict, Iterator, List, Optional, Type, Union, cast
+
+from pendulum import parse as parse_date
+import jmespath  # type: ignore [import-untyped]
+
+from celpy import InterpretedRunner, celtypes
+from celpy.adapter import json_to_cel
+from celpy.evaluation import Annotation, Context, Evaluator
+
+logger = logging.getLogger(f"celpy.{__name__}")
+
+
+
+[docs] +class C7NContext: + """ + Saves current C7N filter for use by functions in this module. + + This is essential for making C7N filter available to *some* of these functions. + + :: + + with C7NContext(filter): + cel_prgm.evaluate(cel_activation) + """ + +
+[docs] + def __init__(self, filter: Any) -> None: + self.filter = filter
+ + +
+[docs] + def __repr__(self) -> str: # pragma: no cover + return f"{self.__class__.__name__}(filter={self.filter!r})"
+ + +
+[docs] + def __enter__(self) -> None: + global C7N + C7N = self
+ + +
+[docs] + def __exit__( + self, + exc_type: Optional[Type[BaseException]], + exc_value: Optional[BaseException], + traceback: Optional[TracebackType], + ) -> None: + global C7N + C7N = cast("C7NContext", None) + return
+
+ + + +# An object used for access to the C7N filter. +# A module global makes the interface functions much simpler. +# They can rely on `C7N.filter` providing the current `CELFilter` instance. +C7N = cast("C7NContext", None) + + +
+[docs] +def key(source: celtypes.ListType, target: celtypes.StringType) -> celtypes.Value: + """ + The C7N shorthand ``tag:Name`` doesn't translate well to CEL. It extracts a single value + from a sequence of objects with a ``{"Key": x, "Value": y}`` structure; specifically, + the value for ``y`` when ``x == "Name"``. + + This function locate a particular "Key": target within a list of {"Key": x, "Value", y} items, + returning the y value if one is found, null otherwise. + + In effect, the ``key()`` function:: + + resource["Tags"].key("Name")["Value"] + + is somewhat like:: + + resource["Tags"].filter(x, x["Key"] == "Name")[0] + + But the ``key()`` function doesn't raise an exception if the key is not found, + instead it returns None. + + We might want to generalize this into a ``first()`` reduction macro. + ``resource["Tags"].first(x, x["Key"] == "Name" ? x["Value"] : null, null)`` + This macro returns the first non-null value or the default (which can be ``null``.) + """ + key = celtypes.StringType("Key") + value = celtypes.StringType("Value") + matches: Iterator[celtypes.Value] = ( + item + for item in source + if cast(celtypes.StringType, cast(celtypes.MapType, item).get(key)) == target + ) + try: + return cast(celtypes.MapType, next(matches)).get(value) + except StopIteration: + return None
+ + + +
+[docs] +def glob(text: celtypes.StringType, pattern: celtypes.StringType) -> celtypes.BoolType: + """Compare a string with a pattern. + + While ``"*.py".glob(some_string)`` seems logical because the pattern the more persistent object, + this seems to cause confusion. + + We use ``some_string.glob("*.py")`` to express a regex-like rule. This parallels the CEL + `.matches()` method. + + We also support ``glob(some_string, "*.py")``. + """ + return celtypes.BoolType(fnmatch.fnmatch(text, pattern))
+ + + +
+[docs] +def difference(left: celtypes.ListType, right: celtypes.ListType) -> celtypes.BoolType: + """ + Compute the difference between two lists. This is ordered set difference: left - right. + It's true if the result is non-empty: there is an item in the left, not present in the right. + It's false if the result is empty: the lists are the same. + """ + return celtypes.BoolType(bool(set(left) - set(right)))
+ + + +
+[docs] +def intersect(left: celtypes.ListType, right: celtypes.ListType) -> celtypes.BoolType: + """ + Compute the intersection between two lists. + It's true if the result is non-empty: there is an item in both lists. + It's false if the result is empty: there is no common item between the lists. + """ + return celtypes.BoolType(bool(set(left) & set(right)))
+ + + +
+[docs] +def normalize(string: celtypes.StringType) -> celtypes.StringType: + """ + Normalize a string. + """ + return celtypes.StringType(string.lower().strip())
+ + + +
+[docs] +def unique_size(collection: celtypes.ListType) -> celtypes.IntType: + """ + Unique size of a list + """ + return celtypes.IntType(len(set(collection)))
+ + + +
+[docs] +class IPv4Network(ipaddress.IPv4Network): + # Override for net 2 net containment comparison +
+[docs] + def __contains__(self, other): # type: ignore[no-untyped-def] + if other is None: + return False + if isinstance(other, ipaddress._BaseNetwork): + return self.supernet_of(other) # type: ignore[no-untyped-call] + return super(IPv4Network, self).__contains__(other)
+ + + contains = __contains__ + + if sys.version_info.major == 3 and sys.version_info.minor <= 6: # pragma: no cover + + @staticmethod + def _is_subnet_of(a, b): # type: ignore[no-untyped-def] + try: + # Always false if one is v4 and the other is v6. + if a._version != b._version: + raise TypeError(f"{a} and {b} are not of the same version") + return ( + b.network_address <= a.network_address + and b.broadcast_address >= a.broadcast_address + ) + except AttributeError: + raise TypeError( + f"Unable to test subnet containment between {a} and {b}" + ) + + def supernet_of(self, other): # type: ignore[no-untyped-def] + """Return True if this network is a supernet of other.""" + return self._is_subnet_of(other, self) # type: ignore[no-untyped-call]
+ + + +CIDR = Union[None, IPv4Network, ipaddress.IPv4Address] +CIDR_Class = Union[Type[IPv4Network], Callable[..., ipaddress.IPv4Address]] + + +
+[docs] +def parse_cidr(value: str) -> CIDR: + """ + Process cidr ranges. + + This is a union of types outside CEL. + + It appears to be Union[None, IPv4Network, ipaddress.IPv4Address] + """ + klass: CIDR_Class = IPv4Network + if "/" not in value: + klass = ipaddress.ip_address # type: ignore[assignment] + v: CIDR + try: + v = klass(value) + except (ipaddress.AddressValueError, ValueError): + v = None + return v
+ + + +
+[docs] +def size_parse_cidr( + value: celtypes.StringType, +) -> Optional[celtypes.IntType]: + """CIDR prefixlen value""" + cidr = parse_cidr(value) + if cidr and isinstance(cidr, IPv4Network): + return celtypes.IntType(cidr.prefixlen) + else: + return None
+ + + +
+[docs] +class ComparableVersion(Version): + """ + The old LooseVersion could fail on comparing present strings, used + in the value as shorthand for certain options. + + The new Version doesn't fail as easily. + """ + +
+[docs] + def __eq__(self, other: object) -> bool: + try: + return super(ComparableVersion, self).__eq__(other) + except TypeError: # pragma: no cover + return False
+
+ + + +
+[docs] +def version( + value: celtypes.StringType, +) -> celtypes.Value: # actually, a ComparableVersion + return cast(celtypes.Value, ComparableVersion(value))
+ + + +
+[docs] +def present( + value: celtypes.StringType, +) -> celtypes.Value: + return cast(celtypes.Value, bool(value))
+ + + +
+[docs] +def absent( + value: celtypes.StringType, +) -> celtypes.Value: + return cast(celtypes.Value, not bool(value))
+ + + +
+[docs] +def text_from( + url: celtypes.StringType, +) -> celtypes.Value: + """ + Read raw text from a URL. This can be expanded to accept S3 or other URL's. + """ + req = urllib.request.Request(url, headers={"Accept-Encoding": "gzip"}) + raw_data: str + with closing(urllib.request.urlopen(req)) as response: + if response.info().get("Content-Encoding") == "gzip": + raw_data = zlib.decompress(response.read(), zlib.MAX_WBITS | 32).decode( + "utf8" + ) + else: + raw_data = response.read().decode("utf-8") + return celtypes.StringType(raw_data)
+ + + +
+[docs] +def parse_text( + source_text: celtypes.StringType, format: celtypes.StringType +) -> celtypes.Value: + """ + Parse raw text using a given format. + """ + if format == "json": + return json_to_cel(json.loads(source_text)) + elif format == "txt": + return celtypes.ListType( + [celtypes.StringType(s.rstrip()) for s in source_text.splitlines()] + ) + elif format in ("ldjson", "ndjson", "jsonl"): + return celtypes.ListType( + [json_to_cel(json.loads(s)) for s in source_text.splitlines()] + ) + elif format == "csv": + return celtypes.ListType( + [json_to_cel(row) for row in csv.reader(io.StringIO(source_text))] + ) + elif format == "csv2dict": + return celtypes.ListType( + [json_to_cel(row) for row in csv.DictReader(io.StringIO(source_text))] + ) + else: + raise ValueError(f"Unsupported format: {format!r}") # pragma: no cover
+ + + +
+[docs] +def value_from( + url: celtypes.StringType, + format: Optional[celtypes.StringType] = None, +) -> celtypes.Value: + """ + Read values from a URL. + + First, do :func:`text_from` to read the source. + Then, do :func:`parse_text` to parse the source, if needed. + + This makes the format optional, and deduces it from the URL's path information. + + C7N will generally replace this with a function + that leverages a more sophisticated :class:`c7n.resolver.ValuesFrom`. + """ + supported_formats = ("json", "ndjson", "ldjson", "jsonl", "txt", "csv", "csv2dict") + + # 1. get format either from arg or URL + if not format: + _, suffix = os.path.splitext(url) + format = celtypes.StringType(suffix[1:]) + if format not in supported_formats: + raise ValueError(f"Unsupported format: {format!r}") + + # 2. read raw data + # Note this is directly bound to text_from() and does not go though the environment + # or other CEL indirection. + raw_data = cast(celtypes.StringType, text_from(url)) + + # 3. parse physical format (json, ldjson, ndjson, jsonl, txt, csv, csv2dict) + return parse_text(raw_data, format)
+ + + +
+[docs] +def jmes_path( + source_data: celtypes.Value, path_source: celtypes.StringType +) -> celtypes.Value: + """ + Apply JMESPath to an object read from from a URL. + """ + expression = jmespath.compile(path_source) + return json_to_cel(expression.search(source_data))
+ + + +
+[docs] +def jmes_path_map( + source_data: celtypes.ListType, path_source: celtypes.StringType +) -> celtypes.ListType: + """ + Apply JMESPath to a each object read from from a URL. + This is for ndjson, nljson and jsonl files. + """ + expression = jmespath.compile(path_source) + return celtypes.ListType( + [json_to_cel(expression.search(row)) for row in source_data] + )
+ + + +
+[docs] +def marked_key( + source: celtypes.ListType, target: celtypes.StringType +) -> celtypes.Value: + """ + Examines a list of {"Key": text, "Value": text} mappings + looking for the given Key value. + + Parses a ``message:action@action_date`` value into a mapping + {"message": message, "action": action, "action_date": action_date} + + If no Key or no Value or the Value isn't the right structure, + the result is a null. + """ + value = key(source, target) + if value is None: + return None + try: + msg, tgt = cast(celtypes.StringType, value).rsplit(":", 1) + action, action_date_str = tgt.strip().split("@", 1) + except ValueError: + return None + return celtypes.MapType( + { + celtypes.StringType("message"): celtypes.StringType(msg), + celtypes.StringType("action"): celtypes.StringType(action), + celtypes.StringType("action_date"): celtypes.TimestampType(action_date_str), + } + )
+ + + +
+[docs] +def image(resource: celtypes.MapType) -> celtypes.Value: + """ + Reach into C7N to get the image details for this EC2 or ASG resource. + + Minimally, the creation date is transformed into a CEL timestamp. + We may want to slightly generalize this to json_to_cell() the entire Image object. + + The following may be usable, but it seems too complex: + + :: + + C7N.filter.prefetch_instance_images(C7N.policy.resources) + image = C7N.filter.get_instance_image(resource["ImageId"]) + return json_to_cel(image) + + .. todo:: Refactor C7N + + Provide the :py:class:`InstanceImageBase` mixin in a :py:class:`CELFilter` class. + We want to have the image details in the new :py:class:`CELFilter` instance. + """ + + # Assuming the :py:class:`CELFilter` class has this method extracted from the legacy filter. + # Requies the policy already did this: C7N.filter.prefetch_instance_images([resource]) to + # populate cache. + image = C7N.filter.get_instance_image(resource) + + if image: + creation_date = image["CreationDate"] + image_name = image["Name"] + else: + creation_date = "2000-01-01T01:01:01.000Z" + image_name = "" + + return json_to_cel({"CreationDate": parse_date(creation_date), "Name": image_name})
+ + + +
+[docs] +def get_raw_metrics(request: celtypes.MapType) -> celtypes.Value: + """ + Reach into C7N and make a statistics request using the current C7N filter object. + + The ``request`` parameter is the request object that is passed through to AWS via + the current C7N filter's manager. The request is a Mapping with the following keys and values: + + :: + + get_raw_metrics({ + "Namespace": "AWS/EC2", + "MetricName": "CPUUtilization", + "Dimensions": {"Name": "InstanceId", "Value": resource.InstanceId}, + "Statistics": ["Average"], + "StartTime": now - duration("4d"), + "EndTime": now, + "Period": duration("86400s") + }) + + The request is passed through to AWS more-or-less directly. The result is a CEL + list of values for then requested statistic. A ``.map()`` macro + can be used to compute additional details. An ``.exists()`` macro can filter the + data to look for actionable values. + + We would prefer to refactor C7N and implement this with code something like this: + + :: + + C7N.filter.prepare_query(C7N.policy.resources) + data = C7N.filter.get_resource_statistics(client, resource) + return json_to_cel(data) + + .. todo:: Refactor C7N + + Provide a :py:class:`MetricsAccess` mixin in a :py:class:`CELFilter` class. + We want to have the metrics processing in the new :py:class:`CELFilter` instance. + + """ + client = C7N.filter.manager.session_factory().client("cloudwatch") + data = client.get_metric_statistics( + Namespace=request["Namespace"], + MetricName=request["MetricName"], + Statistics=request["Statistics"], + StartTime=request["StartTime"], + EndTime=request["EndTime"], + Period=request["Period"], + Dimensions=request["Dimensions"], + )["Datapoints"] + + return json_to_cel(data)
+ + + +
+[docs] +def get_metrics( + resource: celtypes.MapType, request: celtypes.MapType +) -> celtypes.Value: + """ + Reach into C7N and make a statistics request using the current C7N filter. + + This builds a request object that is passed through to AWS via the :func:`get_raw_metrics` + function. + + The ``request`` parameter is a Mapping with the following keys and values: + + :: + + resource.get_metrics({"MetricName": "CPUUtilization", "Statistic": "Average", + "StartTime": now - duration("4d"), "EndTime": now, "Period": duration("86400s")} + ).exists(m, m < 30) + + The namespace is derived from the ``C7N.policy``. The dimensions are derived from + the ``C7N.fiter.model``. + + .. todo:: Refactor C7N + + Provide a :py:class:`MetricsAccess` mixin in a :py:class:`CELFilter` class. + We want to have the metrics processing in the new :py:class:`CELFilter` instance. + + """ + dimension = C7N.filter.manager.get_model().dimension + namespace = C7N.filter.manager.resource_type + # TODO: Varies by resource/policy type. Each policy's model may have different dimensions. + dimensions = [{"Name": dimension, "Value": resource.get(dimension)}] + raw_metrics = get_raw_metrics( + cast( + celtypes.MapType, + json_to_cel( + { + "Namespace": namespace, + "MetricName": request["MetricName"], + "Dimensions": dimensions, + "Statistics": [request["Statistic"]], + "StartTime": request["StartTime"], + "EndTime": request["EndTime"], + "Period": request["Period"], + } + ), + ) + ) + return json_to_cel( + [ + cast(Dict[str, celtypes.Value], item).get(request["Statistic"]) + for item in cast(List[celtypes.Value], raw_metrics) + ] + )
+ + + +
+[docs] +def get_raw_health_events(request: celtypes.MapType) -> celtypes.Value: + """ + Reach into C7N and make a health-events request using the current C7N filter. + + The ``request`` parameter is the filter object that is passed through to AWS via + the current C7N filter's manager. The request is a List of AWS health events. + + :: + + get_raw_health_events({ + "services": ["ELASTICFILESYSTEM"], + "regions": ["us-east-1", "global"], + "eventStatusCodes": ['open', 'upcoming'], + }) + """ + client = C7N.filter.manager.session_factory().client( + "health", region_name="us-east-1" + ) + data = client.describe_events(filter=request)["events"] + return json_to_cel(data)
+ + + +
+[docs] +def get_health_events( + resource: celtypes.MapType, statuses: Optional[List[celtypes.Value]] = None +) -> celtypes.Value: + """ + Reach into C7N and make a health-event request using the current C7N filter. + + This builds a request object that is passed through to AWS via the :func:`get_raw_health_events` + function. + + .. todo:: Handle optional list of event types. + """ + if not statuses: + statuses = [celtypes.StringType("open"), celtypes.StringType("upcoming")] + phd_svc_name_map = { + "app-elb": "ELASTICLOADBALANCING", + "ebs": "EBS", + "efs": "ELASTICFILESYSTEM", + "elb": "ELASTICLOADBALANCING", + "emr": "ELASTICMAPREDUCE", + } + m = C7N.filter.manager + service = phd_svc_name_map.get(m.data["resource"], m.get_model().service.upper()) + raw_events = get_raw_health_events( + cast( + celtypes.MapType, + json_to_cel( + { + "services": [service], + "regions": [m.config.region, "global"], + "eventStatusCodes": statuses, + } + ), + ) + ) + return raw_events
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+[docs] +def security_group( + security_group_id: celtypes.MapType, +) -> celtypes.Value: + """ + Reach into C7N and make a get_related() request using the current C7N filter to get + the security group. + + .. todo:: Refactor C7N + + Provide the :py:class:`RelatedResourceFilter` mixin in a :py:class:`CELFilter` class. + We want to have the related id's details in the new :py:class:`CELFilter` instance. + See :py:class:`VpcSecurityGroupFilter` subclass of :py:class:`RelatedResourceFilter`. + """ + + # Assuming the :py:class:`CELFilter` class has this method extracted from the legacy filter. + security_groups = C7N.filter.get_related([security_group_id]) + return json_to_cel(security_groups)
+ + + +
+[docs] +def subnet( + subnet_id: celtypes.Value, +) -> celtypes.Value: + """ + Reach into C7N and make a get_related() request using the current C7N filter to get + the subnet. + + .. todo:: Refactor C7N + + Provide the :py:class:`RelatedResourceFilter` mixin in a :py:class:`CELFilter` class. + We want to have the related id's details in the new :py:class:`CELFilter` instance. + See :py:class:`VpcSubnetFilter` subclass of :py:class:`RelatedResourceFilter`. + """ + # Get related ID's first, then get items for the related ID's. + subnets = C7N.filter.get_related([subnet_id]) + return json_to_cel(subnets)
+ + + +
+[docs] +def flow_logs( + resource: celtypes.MapType, +) -> celtypes.Value: + """ + Reach into C7N and locate the flow logs using the current C7N filter. + + .. todo:: Refactor C7N + + Provide a separate function to get the flow logs, separate from the + the filter processing. + + .. todo:: Refactor :func:`c7nlib.flow_logs` -- it exposes too much implementation detail. + + """ + # TODO: Refactor into a function in ``CELFilter``. Should not be here. + client = C7N.filter.manager.session_factory().client("ec2") + logs = client.describe_flow_logs().get("FlowLogs", ()) + m = C7N.filter.manager.get_model() + resource_map: Dict[str, List[Dict[str, Any]]] = {} + for fl in logs: + resource_map.setdefault(fl["ResourceId"], []).append(fl) + if resource.get(m.id) in resource_map: + flogs = resource_map[cast(str, resource.get(m.id))] + return json_to_cel(flogs) + return json_to_cel([])
+ + + +
+[docs] +def vpc( + vpc_id: celtypes.Value, +) -> celtypes.Value: + """ + Reach into C7N and make a ``get_related()`` request using the current C7N filter to get + the VPC details. + + .. todo:: Refactor C7N + + Provide the :py:class:`RelatedResourceFilter` mixin in a :py:class:`CELFilter` class. + We want to have the related id's details in the new :py:class:`CELFilter` instance. + See :py:class:`VpcFilter` subclass of :py:class:`RelatedResourceFilter`. + """ + # Assuming the :py:class:`CELFilter` class has this method extracted from the legacy filter. + vpc = C7N.filter.get_related([vpc_id]) + return json_to_cel(vpc)
+ + + +
+[docs] +def subst( + jmes_path: celtypes.StringType, +) -> celtypes.StringType: + """ + Reach into C7N and build a set of substitutions to replace text in a JMES path. + + This is based on how :py:class:`c7n.resolver.ValuesFrom` works. There are + two possible substitution values: + + - account_id + - region + + :param jmes_path: the source + :return: A JMES with values replaced. + """ + + config_args = { + "account_id": C7N.filter.manager.config.account_id, + "region": C7N.filter.manager.config.region, + } + return celtypes.StringType(jmes_path.format(**config_args))
+ + + +
+[docs] +def credentials(resource: celtypes.MapType) -> celtypes.Value: + """ + Reach into C7N and make a get_related() request using the current C7N filter to get + the IAM-role credential details. + + See :py:class:`c7n.resources.iam.CredentialReport` filter. + The `get_credential_report()` function does what we need. + + .. todo:: Refactor C7N + + Refactor the :py:class:`c7n.resources.iam.CredentialReport` filter into a + ``CredentialReportMixin`` mixin to the :py:class:`CELFilter` class. + The ``get_credential_report()`` function does what we need. + """ + return json_to_cel(C7N.filter.get_credential_report(resource))
+ + + +
+[docs] +def kms_alias( + vpc_id: celtypes.Value, +) -> celtypes.Value: + """ + Reach into C7N and make a get_matching_aliases() request using the current C7N filter to get + the alias. + + See :py:class:`c7n.resources.kms.ResourceKmsKeyAlias`. + The `get_matching_aliases()` function does what we need. + + .. todo:: Refactor C7N + + Refactor the :py:class:`c7n.resources.kms.ResourceKmsKeyAlias` filter into a + ``ResourceKmsKeyAliasMixin`` mixin to the :py:class:`CELFilter` class. + The ``get_matching_aliases()`` dfunction does what we need. + """ + return json_to_cel(C7N.filter.get_matching_aliases())
+ + + +
+[docs] +def kms_key( + key_id: celtypes.Value, +) -> celtypes.Value: + """ + Reach into C7N and make a ``get_related()`` request using the current C7N filter to get + the key. We're looking for the c7n.resources.kms.Key resource manager to get the related key. + + .. todo:: Refactor C7N + + Provide the :py:class:`RelatedResourceFilter` mixin in a :py:class:`CELFilter` class. + """ + key = C7N.filter.get_related([key_id]) + return json_to_cel(key)
+ + + +
+[docs] +def resource_schedule( + tag_value: celtypes.Value, +) -> celtypes.Value: + """ + Reach into C7N and use the the :py:class:`c7n.filters.offhours.ScheduleParser` class + to examine the tag's value, the current time, and return a True/False. + This parser is the `parser` value of the :py:class:`c7n.filters.offhours.Time` filter class. + Using the filter's `parser.parse(value)` provides needed structure. + + The `filter.parser.parse(value)` will transform text of the Tag value + into a dictionary. This is further transformed to something we can use in CEL. + + From this + :: + + off=[(M-F,21),(U,18)];on=[(M-F,6),(U,10)];tz=pt + + C7N ScheduleParser produces this + :: + + { + off: [ + { days: [1, 2, 3, 4, 5], hour: 21 }, + { days: [0], hour: 18 } + ], + on: [ + { days: [1, 2, 3, 4, 5], hour: 6 }, + { days: [0], hour: 10 } + ], + tz: "pt" + } + + For CEL, we need this + :: + + { + off: [ + { days: [1, 2, 3, 4, 5], hour: 21, tz: "pt" }, + { days: [0], hour: 18, tz: "pt" } + ], + on: [ + { days: [1, 2, 3, 4, 5], hour: 6, tz: "pt" }, + { days: [0], hour: 10, tz: "pt" } + ], + } + + This lets a CEL expression use + :: + + key("maid_offhours").resource_schedule().off.exists(s, + now.getDayOfWeek(s.tz) in s.days && now.getHour(s.tz) == s.hour) + """ + c7n_sched_doc = C7N.filter.parser.parse(tag_value) + tz = c7n_sched_doc.pop("tz", "et") + cel_sched_doc = { + state: [ + {"days": time["days"], "hour": time["hour"], "tz": tz} for time in time_list + ] + for state, time_list in c7n_sched_doc.items() + } + return json_to_cel(cel_sched_doc)
+ + + +
+[docs] +def get_accounts( + resource: celtypes.MapType, +) -> celtypes.Value: + """ + Reach into C7N filter and get accounts for a given resource. + Used by resources like AMI's, log-groups, ebs-snapshot, etc. + + .. todo:: Refactor C7N + + Provide the :py:class:`c7n.filters.iamaccessfilter.CrossAccountAccessFilter` + as a mixin to ``CELFilter``. + """ + return json_to_cel(C7N.filter.get_accounts())
+ + + +
+[docs] +def get_vpcs( + resource: celtypes.MapType, +) -> celtypes.Value: + """ + Reach into C7N filter and get vpcs for a given resource. + Used by resources like AMI's, log-groups, ebs-snapshot, etc. + + .. todo:: Refactor C7N + + Provide the :py:class:`c7n.filters.iamaccessfilter.CrossAccountAccessFilter` + as a mixin to ``CELFilter``. + """ + return json_to_cel(C7N.filter.get_vpcs())
+ + + +
+[docs] +def get_vpces( + resource: celtypes.MapType, +) -> celtypes.Value: + """ + Reach into C7N filter and get vpces for a given resource. + Used by resources like AMI's, log-groups, ebs-snapshot, etc. + + .. todo:: Refactor C7N + + Provide the :py:class:`c7n.filters.iamaccessfilter.CrossAccountAccessFilter` + as a mixin to ``CELFilter``. + + """ + return json_to_cel(C7N.filter.get_vpces())
+ + + +
+[docs] +def get_orgids( + resource: celtypes.MapType, +) -> celtypes.Value: + """ + Reach into C7N filter and get orgids for a given resource. + Used by resources like AMI's, log-groups, ebs-snapshot, etc. + + .. todo:: Refactor C7N + + Provide the :py:class:`c7n.filters.iamaccessfilter.CrossAccountAccessFilter` + as a mixin to ``CELFilter``. + """ + return json_to_cel(C7N.filter.get_orgids())
+ + + +
+[docs] +def get_endpoints( + resource: celtypes.MapType, +) -> celtypes.Value: + """For sns resources + + .. todo:: Refactor C7N + + Provide the :py:class:`c7n.filters.iamaccessfilter.CrossAccountAccessFilter` + as a mixin to ``CELFilter``. + """ + return json_to_cel(C7N.filter.get_endpoints())
+ + + +
+[docs] +def get_protocols( + resource: celtypes.MapType, +) -> celtypes.Value: + """For sns resources + + .. todo:: Refactor C7N + """ + return json_to_cel(C7N.filter.get_protocols())
+ + + +
+[docs] +def get_key_policy( + resource: celtypes.MapType, +) -> celtypes.Value: + """For kms resources + + .. todo:: Refactor C7N + """ + key_id = resource.get( + celtypes.StringType("TargetKeyId"), resource.get(celtypes.StringType("KeyId")) + ) + client = C7N.filter.manager.session_factory().client("kms") + return json_to_cel( + client.get_key_policy(KeyId=key_id, PolicyName="default")["Policy"] + )
+ + + +
+[docs] +def get_resource_policy( + resource: celtypes.MapType, +) -> celtypes.Value: + """ + Reach into C7N filter and get the resource policy for a given resource. + Used by resources like AMI's, log-groups, ebs-snapshot, etc. + + .. todo:: Refactor C7N + """ + return json_to_cel(C7N.filter.get_resource_policy())
+ + + +
+[docs] +def describe_subscription_filters( + resource: celtypes.MapType, +) -> celtypes.Value: + """ + For log-groups resources. + + .. todo:: Refactor C7N + + this should be directly available in CELFilter. + """ + client = C7N.filter.manager.session_factory().client("logs") + return json_to_cel( + C7N.filter.manager.retry( + client.describe_subscription_filters, logGroupName=resource["logGroupName"] + ).get("subscriptionFilters", ()) + )
+ + + +
+[docs] +def describe_db_snapshot_attributes( + resource: celtypes.MapType, +) -> celtypes.Value: + """ + For rds-snapshot and ebs-snapshot resources + + .. todo:: Refactor C7N + + this should be directly available in CELFilter. + """ + client = C7N.filter.manager.session_factory().client("ec2") + return json_to_cel( + C7N.filter.manager.retry( + client.describe_snapshot_attribute, + SnapshotId=resource["SnapshotId"], + Attribute="createVolumePermission", + ) + )
+ + + +
+[docs] +def arn_split(arn: celtypes.StringType, field: celtypes.StringType) -> celtypes.Value: + """ + Parse an ARN, removing a partivular field. + The field name must one one of + "partition", "service", "region", "account-id", "resource-type", "resource-id" + In the case of a ``resource-type/resource-id`` path, this will be a "resource-id" value, + and there will be no "resource-type". + + Examples formats + + ``arn:partition:service:region:account-id:resource-id`` + + ``arn:partition:service:region:account-id:resource-type/resource-id`` + + ``arn:partition:service:region:account-id:resource-type:resource-id`` + """ + field_names = { + len(names): names + for names in [ + ("partition", "service", "region", "account-id", "resource-id"), + ( + "partition", + "service", + "region", + "account-id", + "resource-type", + "resource-id", + ), + ] + } + prefix, *fields = arn.split(":") + if prefix != "arn": + raise ValueError(f"Not an ARN: {arn}") + mapping = dict(zip(field_names[len(fields)], fields)) + return json_to_cel(mapping[field])
+ + + +
+[docs] +def all_images() -> celtypes.Value: + """ + Depends on :py:meth:`CELFilter._pull_ec2_images` and :py:meth:`CELFilter._pull_asg_images` + + See :py:class:`c7n.resources.ami.ImageUnusedFilter` + """ + return json_to_cel( + list(C7N.filter._pull_ec2_images() | C7N.filter._pull_asg_images()) + )
+ + + +
+[docs] +def all_snapshots() -> celtypes.Value: + """ + Depends on :py:meth:`CELFilter._pull_asg_snapshots` + and :py:meth:`CELFilter._pull_ami_snapshots` + + See :py:class:`c7n.resources.ebs.SnapshotUnusedFilter` + """ + return json_to_cel( + list(C7N.filter._pull_asg_snapshots() | C7N.filter._pull_ami_snapshots()) + )
+ + + +
+[docs] +def all_launch_configuration_names() -> celtypes.Value: + """ + Depends on :py:meth:`CELFilter.manager.get_launch_configuration_names` + + See :py:class:`c7n.resources.asg.UnusedLaunchConfig` + """ + asgs = C7N.filter.manager.get_resource_manager("asg").resources() + used = set( + [ + a.get("LaunchConfigurationName", a["AutoScalingGroupName"]) + for a in asgs + if not a.get("LaunchTemplate") + ] + ) + return json_to_cel(list(used))
+ + + +
+[docs] +def all_service_roles() -> celtypes.Value: + """ + Depends on :py:meth:`CELFilter.service_role_usage` + + See :py:class:`c7n.resources.iam.UnusedIamRole` + """ + return json_to_cel(C7N.filter.service_role_usage())
+ + + +
+[docs] +def all_instance_profiles() -> celtypes.Value: + """ + Depends on :py:meth:`CELFilter.instance_profile_usage` + + See :py:class:`c7n.resources.iam.UnusedInstanceProfiles` + """ + return json_to_cel(C7N.filter.instance_profile_usage())
+ + + +
+[docs] +def all_dbsubenet_groups() -> celtypes.Value: + """ + Depends on :py:meth:`CELFilter.get_dbsubnet_group_used` + + See :py:class:`c7n.resources.rds.UnusedRDSSubnetGroup` + """ + rds = C7N.filter.manager.get_resource_manager("rds").resources() + used = set([r.get("DBSubnetGroupName", r["DBInstanceIdentifier"]) for r in rds]) + return json_to_cel(list(used))
+ + + +
+[docs] +def all_scan_groups() -> celtypes.Value: + """ + Depends on :py:meth:`CELFilter.scan_groups` + + See :py:class:`c7n.resources.vpc.UnusedSecurityGroup` + """ + return json_to_cel(C7N.filter.scan_groups())
+ + + +
+[docs] +def get_access_log(resource: celtypes.MapType) -> celtypes.Value: + """ + Depends on :py:meth:`CELFilter.resources` + + See :py:class:`c7n.resources.elb.IsNotLoggingFilter` and + :py:class:`c7n.resources.elb.IsLoggingFilter`. + """ + client = C7N.filter.manager.session_factory().client("elb") + results = client.describe_load_balancer_attributes( + LoadBalancerName=resource["LoadBalancerName"] + ) + return json_to_cel(results["LoadBalancerAttributes"])
+ + + +
+[docs] +def get_load_balancer(resource: celtypes.MapType) -> celtypes.Value: + """ + Depends on :py:meth:`CELFilter.resources` + + See :py:class:`c7n.resources.appelb.IsNotLoggingFilter` and + :py:class:`c7n.resources.appelb.IsLoggingFilter`. + """ + + def parse_attribute_value(v: str) -> Union[int, bool, str]: + """Lightweight JSON atomic value convertion to native Python.""" + if v.isdigit(): + return int(v) + elif v == "true": + return True + elif v == "false": + return False + return v + + client = C7N.filter.manager.session_factory().client("elbv2") + results = client.describe_load_balancer_attributes( + LoadBalancerArn=resource["LoadBalancerArn"] + ) + # print(results) + return json_to_cel( + dict( + (item["Key"], parse_attribute_value(item["Value"])) + for item in results["Attributes"] + ) + )
+ + + +
+[docs] +def shield_protection(resource: celtypes.MapType) -> celtypes.Value: + """ + Depends on the :py:meth:`c7n.resources.shield.IsShieldProtected.process` method. + This needs to be refactored and renamed to avoid collisions with other ``process()`` variants. + + Applies to most resource types. + """ + client = C7N.filter.manager.session_factory().client( + "shield", region_name="us-east-1" + ) + protections = C7N.filter.get_type_protections( + client, C7N.filter.manager.get_model() + ) + protected_resources = [p["ResourceArn"] for p in protections] + return json_to_cel(protected_resources)
+ + + +
+[docs] +def shield_subscription(resource: celtypes.MapType) -> celtypes.Value: + """ + Depends on :py:meth:`c7n.resources.account.ShieldEnabled.process` method. + This needs to be refactored and renamed to avoid collisions with other ``process()`` variants. + + Applies to account resources only. + """ + subscriptions = C7N.filter.account_shield_subscriptions(resource) + return json_to_cel(subscriptions)
+ + + +
+[docs] +def web_acls(resource: celtypes.MapType) -> celtypes.Value: + """ + Depends on :py:meth:`c7n.resources.cloudfront.IsWafEnabled.process` method. + This needs to be refactored and renamed to avoid collisions with other ``process()`` variants. + """ + wafs = C7N.filter.manager.get_resource_manager("waf").resources() + waf_name_id_map = {w["Name"]: w["WebACLId"] for w in wafs} + return json_to_cel(waf_name_id_map)
+ + + +DECLARATIONS: Dict[str, Annotation] = { + "glob": celtypes.FunctionType, + "difference": celtypes.FunctionType, + "intersect": celtypes.FunctionType, + "normalize": celtypes.FunctionType, + "parse_cidr": celtypes.FunctionType, # Callable[..., CIDR], + "size_parse_cidr": celtypes.FunctionType, + "unique_size": celtypes.FunctionType, + "version": celtypes.FunctionType, # Callable[..., ComparableVersion], + "present": celtypes.FunctionType, + "absent": celtypes.FunctionType, + "text_from": celtypes.FunctionType, + "value_from": celtypes.FunctionType, + "jmes_path": celtypes.FunctionType, + "jmes_path_map": celtypes.FunctionType, + "key": celtypes.FunctionType, + "marked_key": celtypes.FunctionType, + "image": celtypes.FunctionType, + "get_metrics": celtypes.FunctionType, + "get_related_ids": celtypes.FunctionType, + "security_group": celtypes.FunctionType, + "subnet": celtypes.FunctionType, + "flow_logs": celtypes.FunctionType, + "vpc": celtypes.FunctionType, + "subst": celtypes.FunctionType, + "credentials": celtypes.FunctionType, + "kms_alias": celtypes.FunctionType, + "kms_key": celtypes.FunctionType, + "resource_schedule": celtypes.FunctionType, + "get_accounts": celtypes.FunctionType, + "get_related_sgs": celtypes.FunctionType, + "get_related_subnets": celtypes.FunctionType, + "get_related_nat_gateways": celtypes.FunctionType, + "get_related_igws": celtypes.FunctionType, + "get_related_security_configs": celtypes.FunctionType, + "get_related_vpc": celtypes.FunctionType, + "get_related_kms_keys": celtypes.FunctionType, + "get_vpcs": celtypes.FunctionType, + "get_vpces": celtypes.FunctionType, + "get_orgids": celtypes.FunctionType, + "get_endpoints": celtypes.FunctionType, + "get_protocols": celtypes.FunctionType, + "get_key_policy": celtypes.FunctionType, + "get_resource_policy": celtypes.FunctionType, + "describe_subscription_filters": celtypes.FunctionType, + "describe_db_snapshot_attributes": celtypes.FunctionType, + "arn_split": celtypes.FunctionType, + "all_images": celtypes.FunctionType, + "all_snapshots": celtypes.FunctionType, + "all_launch_configuration_names": celtypes.FunctionType, + "all_service_roles": celtypes.FunctionType, + "all_instance_profiles": celtypes.FunctionType, + "all_dbsubenet_groups": celtypes.FunctionType, + "all_scan_groups": celtypes.FunctionType, + "get_access_log": celtypes.FunctionType, + "get_load_balancer": celtypes.FunctionType, + "shield_protection": celtypes.FunctionType, + "shield_subscription": celtypes.FunctionType, + "web_acls": celtypes.FunctionType, + # "etc.": celtypes.FunctionType, +} + +ExtFunction = Callable[..., celtypes.Value] + +FUNCTIONS: Dict[str, ExtFunction] = { + f.__name__: cast(ExtFunction, f) + for f in [ + glob, + difference, + intersect, + normalize, + parse_cidr, + size_parse_cidr, + unique_size, + version, + present, + absent, + text_from, + value_from, + jmes_path, + jmes_path_map, + key, + marked_key, + image, + get_metrics, + get_related_ids, + security_group, + subnet, + flow_logs, + vpc, + subst, + credentials, + kms_alias, + kms_key, + resource_schedule, + get_accounts, + get_related_sgs, + get_related_subnets, + get_related_nat_gateways, + get_related_igws, + get_related_security_configs, + get_related_vpc, + get_related_kms_keys, + get_vpcs, + get_vpces, + get_orgids, + get_endpoints, + get_protocols, + get_key_policy, + get_resource_policy, + describe_subscription_filters, + describe_db_snapshot_attributes, + arn_split, + all_images, + all_snapshots, + all_launch_configuration_names, + all_service_roles, + all_instance_profiles, + all_dbsubenet_groups, + all_scan_groups, + get_access_log, + get_load_balancer, + shield_protection, + shield_subscription, + web_acls, + # etc. + ] +} + + +
+[docs] +class C7N_Interpreted_Runner(InterpretedRunner): + """ + Extends the Evaluation to introduce the C7N CELFilter instance into the evaluation. + + The variable is global to allow the functions to have the simple-looking argument + values that CEL expects. This allows a function in this module to reach outside CEL for + access to C7N's caches. + + .. todo: Refactor to be a mixin to the Runner class hierarchy. + """ + +
+[docs] + def evaluate( + self, context: Context, filter: Optional[Any] = None + ) -> celtypes.Value: + e = Evaluator( + ast=self.ast, + activation=self.new_activation(), + ) + with C7NContext(filter=filter): + value = e.evaluate(context) + return value
+
+ +
+ +
+ +
+
+ +
+
+ + + + + + + \ No newline at end of file diff --git a/docs/build/html/_modules/celpy/celparser.html b/docs/build/html/_modules/celpy/celparser.html index 2a3511c..24431b9 100644 --- a/docs/build/html/_modules/celpy/celparser.html +++ b/docs/build/html/_modules/celpy/celparser.html @@ -47,19 +47,19 @@

Source code for celpy.celparser

 # See the License for the specific language governing permissions and limitations under the License.
 
 """
-CEL Parser.
+A **Facade** around the CEL parser.
 
-See  https://github.com/google/cel-spec/blob/master/doc/langdef.md
+The Parser is an instance of the :py:class:`lark.Lark` class.
 
-https://github.com/google/cel-cpp/blob/master/parser/Cel.g4
+The grammar is in the  ``cel.lark`` file.
 
-https://github.com/google/cel-go/blob/master/parser/gen/CEL.g4
+For more information on CEL syntax, see the following:
 
-Builds a parser from the supplied cel.lark grammar.
+-   https://github.com/google/cel-spec/blob/master/doc/langdef.md
 
-..  todo:: Consider embedding the ``cel.lark`` file as a triple-quoted literal.
+-   https://github.com/google/cel-cpp/blob/master/parser/Cel.g4
 
-    This means fixing a LOT of \\'s. But it also eliminates a data file from the installation.
+-   https://github.com/google/cel-go/blob/master/parser/gen/CEL.g4
 
 Example::
 
@@ -106,6 +106,8 @@ 

Source code for celpy.celparser

 
[docs] class CELParseError(Exception): + """A syntax error in the CEL expression.""" +
[docs] def __init__( @@ -121,13 +123,31 @@

Source code for celpy.celparser

 
[docs] class CELParser: - """Wrapper for the CEL parser and the syntax error messages.""" + """ + Creates a Lark parser with the required options. + + .. important:: **Singleton** + + There is one CEL_PARSER instance created by this class. + This is an optimization for environments like C7N where numerous + CEL expressions may parsed. + + :: + + CELParse.CEL_PARSER = None + + Is required to create another parser instance. + This is commonly required in test environments. + + This is also an **Adapter** for the CEL parser to provide pleasant + syntax error messages. + """ CEL_PARSER: Optional[Lark] = None
[docs] - def __init__(self) -> None: + def __init__(self, tree_class: type = lark.Tree) -> None: if CELParser.CEL_PARSER is None: CEL_grammar = (Path(__file__).parent / "cel.lark").read_text() CELParser.CEL_PARSER = Lark( @@ -140,6 +160,7 @@

Source code for celpy.celparser

                 propagate_positions=True,
                 maybe_placeholders=False,
                 priority="invert",
+                tree_class=tree_class,
             )
@@ -589,14 +610,15 @@

CEL in Python

Navigation

-

Contents:

+

Documentation Content:

diff --git a/docs/build/html/_modules/celpy/celtypes.html b/docs/build/html/_modules/celpy/celtypes.html index 2c975c9..679d139 100644 --- a/docs/build/html/_modules/celpy/celtypes.html +++ b/docs/build/html/_modules/celpy/celtypes.html @@ -47,7 +47,7 @@

Source code for celpy.celtypes

 # See the License for the specific language governing permissions and limitations under the License.
 
 """
-CEL Types: wrappers on Python types to provide CEL semantics.
+Provides wrappers over Python types to provide CEL semantics.
 
 This can be used by a Python module to work with CEL-friendly values and CEL results.
 
@@ -323,12 +323,19 @@ 

Source code for celpy.celtypes

         >>> logical_condition(
         ... BoolType(False), StringType("Not This"), StringType("that"))
         StringType('that')
+
+    .. TODO:: Consider passing closures instead of Values.
+
+        The function can evaluate e().
+        If it's True, return x().
+        If it's False, return y().
+        Otherwise, it's a CELEvalError, which is the result
     """
     if not isinstance(e, BoolType):
         raise TypeError(f"Unexpected {type(e)} ? {type(x)} : {type(y)}")
-    result = x if e else y
-    logger.debug("logical_condition(%r, %r, %r) = %r", e, x, y, result)
-    return result
+ result_value = x if e else y + logger.debug("logical_condition(%r, %r, %r) = %r", e, x, y, result_value) + return result_value
@@ -338,6 +345,12 @@

Source code for celpy.celtypes

     """
     Native Python has a left-to-right rule.
     CEL && is commutative with non-Boolean values, including error objects.
+
+    .. TODO:: Consider passing closures instead of Values.
+
+        The function can evaluate x().
+        If it's False, then return False.
+        Otherwise, it's True or a CELEvalError, return y().
     """
     if not isinstance(x, BoolType) and not isinstance(y, BoolType):
         raise TypeError(f"{type(x)} {x!r} and {type(y)} {y!r}")
@@ -360,14 +373,17 @@ 

Source code for celpy.celtypes

 [docs]
 def logical_not(x: Value) -> Value:
     """
-    Native python `not` isn't fully exposed for CEL types.
+    A function for native python `not`.
+
+    This could almost be `logical_or = evaluation.boolean(operator.not_)`,
+    but the definition would expose Python's notion of "truthiness", which isn't appropriate for CEL.
     """
     if isinstance(x, BoolType):
-        result = BoolType(not x)
+        result_value = BoolType(not x)
     else:
         raise TypeError(f"not {type(x)}")
-    logger.debug("logical_not(%r) = %r", x, result)
-    return result
+ logger.debug("logical_not(%r) = %r", x, result_value) + return result_value
@@ -375,8 +391,8 @@

Source code for celpy.celtypes

 [docs]
 def logical_or(x: Value, y: Value) -> Value:
     """
-    Native Python has a left-to-right rule: (True or y) is True, (False or y) is y.
-    CEL || is commutative with non-Boolean values, including errors.
+    Native Python has a left-to-right rule: ``(True or y)`` is True, ``(False or y)`` is y.
+    CEL ``||`` is commutative with non-Boolean values, including errors.
     ``(x || false)`` is ``x``, and ``(false || y)`` is ``y``.
 
     Example 1::
@@ -391,7 +407,13 @@ 

Source code for celpy.celtypes

 
     is a "True"
 
-    If the operand(s) are not BoolType, we'll create an TypeError that will become a CELEvalError.
+    If the operand(s) are not ``BoolType``, we'll create an ``TypeError`` that will become a ``CELEvalError``.
+
+    .. TODO:: Consider passing closures instead of Values.
+
+        The function can evaluate x().
+        If it's True, then return True.
+        Otherwise, it's False or a CELEvalError, return y().
     """
     if not isinstance(x, BoolType) and not isinstance(y, BoolType):
         raise TypeError(f"{type(x)} {x!r} or {type(y)} {y!r}")
@@ -414,9 +436,9 @@ 

Source code for celpy.celtypes

 [docs]
 class BoolType(int):
     """
-    Native Python permits unary operators on Booleans.
+    Native Python permits all unary operators to work on ``bool`` objects.
 
-    For CEL, We need to prevent -false from working.
+    For CEL, we need to prevent the CEL expression ``-false`` from working.
     """
 
 
@@ -480,7 +502,7 @@

Source code for celpy.celtypes

         elif isinstance(source, MessageType):
             return super().__new__(
                 cls,
-                cast(bytes, source.get(StringType("value"))),  # type: ignore [attr-defined]
+                cast(bytes, source.get(StringType("value"))),
             )
         elif isinstance(source, Iterable):
             return super().__new__(cls, cast(Iterable[int], source))
@@ -603,9 +625,9 @@ 

Source code for celpy.celtypes

 
     @wraps(operator)
     def clamped_operator(*args: Any, **kwargs: Any) -> int:
-        result: int = operator(*args, **kwargs)
-        if -(2**63) <= result < 2**63:
-            return result
+        result_value: int = operator(*args, **kwargs)
+        if -(2**63) <= result_value < 2**63:
+            return result_value
         raise ValueError("overflow")
 
     return cast(IntOperator, clamped_operator)
@@ -832,9 +854,9 @@

Source code for celpy.celtypes

 
     @wraps(operator)
     def clamped_operator(*args: Any, **kwargs: Any) -> int:
-        result = operator(*args, **kwargs)
-        if 0 <= result < 2**64:
-            return result
+        result_value = operator(*args, **kwargs)
+        if 0 <= result_value < 2**64:
+            return result_value
         raise ValueError("overflow")
 
     return cast(IntOperator, clamped_operator)
@@ -1081,14 +1103,14 @@

Source code for celpy.celtypes

             except TypeError as ex:
                 return cast(BoolType, ex)  # Instead of Union[BoolType, TypeError]
 
-        result = len(self) == len(other) and reduce(  # noqa: W503
+        result_value = len(self) == len(other) and reduce(
             logical_and,  # type: ignore [arg-type]
             (equal(item_s, item_o) for item_s, item_o in zip(self, other)),
             BoolType(True),  # type: ignore [arg-type]
         )
-        if isinstance(result, TypeError):
-            raise result
-        return bool(result)
+ if isinstance(result_value, TypeError): + raise result_value + return bool(result_value)
@@ -1103,14 +1125,20 @@

Source code for celpy.celtypes

             except TypeError as ex:
                 return cast(BoolType, ex)  # Instead of Union[BoolType, TypeError]
 
-        result = len(self) != len(other) or reduce(  # noqa: W503
+        result_value = len(self) != len(other) or reduce(
             logical_or,  # type: ignore [arg-type]
             (not_equal(item_s, item_o) for item_s, item_o in zip(self, other)),
             BoolType(False),  # type: ignore [arg-type]
         )
-        if isinstance(result, TypeError):
-            raise result
-        return bool(result)
+ if isinstance(result_value, TypeError): + raise result_value + return bool(result_value)
+ + +
+[docs] + def contains(self, item: Value) -> BoolType: + return BoolType(item in self)
@@ -1145,7 +1173,10 @@

Source code for celpy.celtypes

         if items is None:
             pass
         elif isinstance(items, Sequence):
+            # Must Be Unique
             for name, value in items:
+                if name in self:
+                    raise ValueError(f"Duplicate key {name!r}")
                 self[name] = value
         elif isinstance(items, Mapping):
             for name, value in items.items():
@@ -1182,14 +1213,14 @@ 

Source code for celpy.celtypes

 
         keys_s = self.keys()
         keys_o = other.keys()
-        result = keys_s == keys_o and reduce(  # noqa: W503
+        result_value = keys_s == keys_o and reduce(
             logical_and,  # type: ignore [arg-type]
             (equal(self[k], other[k]) for k in keys_s),
             BoolType(True),  # type: ignore [arg-type]
         )
-        if isinstance(result, TypeError):
-            raise result
-        return bool(result)
+ if isinstance(result_value, TypeError): + raise result_value + return bool(result_value)
@@ -1213,14 +1244,28 @@

Source code for celpy.celtypes

 
         keys_s = self.keys()
         keys_o = other.keys()
-        result = keys_s != keys_o or reduce(  # noqa: W503
+        result_value = keys_s != keys_o or reduce(
             logical_or,  # type: ignore [arg-type]
             (not_equal(self[k], other[k]) for k in keys_s),
             BoolType(False),  # type: ignore [arg-type]
         )
-        if isinstance(result, TypeError):
-            raise result
-        return bool(result)
+ if isinstance(result_value, TypeError): + raise result_value + return bool(result_value)
+ + +
+[docs] + def get(self, key: Any, default: Optional[Any] = None) -> Value: + """There is no default provision in CEL, that's a Python feature.""" + if not MapType.valid_key_type(key): + raise TypeError(f"unsupported key type: {type(key)}") + if key in self: + return super().get(key) + elif default is not None: + return cast(Value, default) + else: + raise KeyError(key)
@@ -1229,6 +1274,12 @@

Source code for celpy.celtypes

     def valid_key_type(key: Any) -> bool:
         """Valid CEL key types. Plus native str for tokens in the source when evaluating ``e.f``"""
         return isinstance(key, (IntType, UintType, BoolType, StringType, str))
+ + +
+[docs] + def contains(self, item: Value) -> BoolType: + return BoolType(item in self)
@@ -1299,6 +1350,12 @@

Source code for celpy.celtypes

 [docs]
     def __hash__(self) -> int:
         return super().__hash__()
+ + +
+[docs] + def contains(self, item: Value) -> BoolType: + return BoolType(cast(StringType, item) in self)
@@ -1392,7 +1449,7 @@

Source code for celpy.celtypes

             return ts
 
         elif isinstance(source, str):
-            # Use dateutil to try a variety of text formats.
+            # Use ``pendulum`` to try a variety of text formats.
             parsed_datetime = cast(datetime.datetime, pendulum.parse(source))
             return super().__new__(
                 cls,
@@ -1429,20 +1486,20 @@ 

Source code for celpy.celtypes

 [docs]
     def __add__(self, other: Any) -> "TimestampType":
         """Timestamp + Duration -> Timestamp"""
-        result = super().__add__(other)
-        if result == NotImplemented:
+        result_value = super().__add__(other)
+        if result_value == NotImplemented:
             return NotImplemented
-        return TimestampType(result)
+ return TimestampType(result_value)
[docs] def __radd__(self, other: Any) -> "TimestampType": """Duration + Timestamp -> Timestamp""" - result = super().__radd__(other) - if result == NotImplemented: + result_value = super().__radd__(other) + if result_value == NotImplemented: return NotImplemented - return TimestampType(result)
+ return TimestampType(result_value)
# For more information, check the typeshed definition @@ -1459,12 +1516,12 @@

Source code for celpy.celtypes

     def __sub__(
         self, other: Union["TimestampType", "DurationType"]
     ) -> Union["TimestampType", "DurationType"]:
-        result = super().__sub__(other)
-        if result == NotImplemented:
-            return cast(DurationType, result)
-        if isinstance(result, datetime.timedelta):
-            return DurationType(result)
-        return TimestampType(result)
+ result_value = super().__sub__(other) + if result_value == NotImplemented: + return cast(DurationType, result_value) + if isinstance(result_value, datetime.timedelta): + return DurationType(result_value) + return TimestampType(result_value)
@@ -1472,7 +1529,7 @@

Source code for celpy.celtypes

     @classmethod
     def tz_name_lookup(cls, tz_name: str) -> Optional[datetime.tzinfo]:
         """
-        The :py:func:`dateutil.tz.gettz` may be extended with additional aliases.
+        The ``pendulum`` parsing may be extended with additional aliases.
 
         ..  TODO: Permit an extension into the timezone lookup.
             Tweak ``celpy.celtypes.TimestampType.TZ_ALIASES``.
@@ -1714,13 +1771,13 @@ 

Source code for celpy.celtypes

         A duration + timestamp is not implemented by the timedelta superclass;
         it is handled by the datetime superclass that implementes timestamp + duration.
         """
-        result = super().__add__(other)
-        if result == NotImplemented:
-            return cast(DurationType, result)
+        result_value = super().__add__(other)
+        if result_value == NotImplemented:
+            return cast(DurationType, result_value)
         # This is handled by TimestampType; this is here for completeness, but isn't used.
-        if isinstance(result, (datetime.datetime, TimestampType)):
-            return TimestampType(result)  # pragma: no cover
-        return DurationType(result)
+ if isinstance(result_value, (datetime.datetime, TimestampType)): + return TimestampType(result_value) # pragma: no cover + return DurationType(result_value)
@@ -1731,13 +1788,13 @@

Source code for celpy.celtypes

 
         Most cases are handled by TimeStamp.
         """
-        result = super().__radd__(other)
-        if result == NotImplemented:
-            return cast(DurationType, result)
+        result_value = super().__radd__(other)
+        if result_value == NotImplemented:
+            return cast(DurationType, result_value)
         # This is handled by TimestampType; this is here for completeness, but isn't used.
-        if isinstance(result, (datetime.datetime, TimestampType)):
-            return TimestampType(result)
-        return DurationType(result)
+ if isinstance(result_value, (datetime.datetime, TimestampType)): + return TimestampType(result_value) + return DurationType(result_value)
@@ -1836,93 +1893,21 @@

Source code for celpy.celtypes

 
- # def get(self, field: Any, default: Optional[Value] = None) -> Value: - # """ - # Alternative implementation with descent to locate a deeply-buried field. - # It seemed like this was the defined behavior. It turns it, it isn't. - # The code is here in case we're wrong and it really is the defined behavior. - # - # Note. There is no default provision in CEL. - # """ - # if field in self: - # return super().get(field) - # - # def descend(message: MessageType, field: Value) -> MessageType: - # if field in message: - # return message - # for k in message.keys(): - # found = descend(message[k], field) - # if found is not None: - # return found - # return None - # - # sub_message = descend(self, field) - # if sub_message is None: - # return default - # return sub_message.get(field) -
[docs] -class TypeType: +class TypeType(type): """ - Annotation used to mark protobuf type objects. - We map these to CELTypes so that type name testing works. + This is primarily used as a function to extract type from an object or a type. + For consistence, we define it as a type so other types can extend it. """ - type_name_mapping = { - "google.protobuf.Duration": DurationType, - "google.protobuf.Timestamp": TimestampType, - "google.protobuf.Int32Value": IntType, - "google.protobuf.Int64Value": IntType, - "google.protobuf.UInt32Value": UintType, - "google.protobuf.UInt64Value": UintType, - "google.protobuf.FloatValue": DoubleType, - "google.protobuf.DoubleValue": DoubleType, - "google.protobuf.Value": MessageType, - "google.protubuf.Any": MessageType, # Weird. - "google.protobuf.Any": MessageType, - "list_type": ListType, - "map_type": MapType, - "map": MapType, - "list": ListType, - "string": StringType, - "bytes": BytesType, - "bool": BoolType, - "int": IntType, - "uint": UintType, - "double": DoubleType, - "null_type": type(None), - "STRING": StringType, - "BOOL": BoolType, - "INT64": IntType, - "UINT64": UintType, - "INT32": IntType, - "UINT32": UintType, - "BYTES": BytesType, - "DOUBLE": DoubleType, - } - -
-[docs] - def __init__(self, value: Any = "") -> None: - if isinstance(value, str) and value in self.type_name_mapping: - self.type_reference = self.type_name_mapping[value] - elif isinstance(value, str): - try: - self.type_reference = eval(value) - except (NameError, SyntaxError): - raise TypeError(f"Unknown type {value!r}") - else: - self.type_reference = value.__class__
- - -
-[docs] - def __eq__(self, other: Any) -> bool: - return ( - other == self.type_reference or isinstance(other, self.type_reference) # noqa: W503 - )
+
+[docs] + def __new__(typ, instance: Any) -> type: + if type(instance) is type: + return typ + return type(instance)
@@ -1952,14 +1937,15 @@

CEL in Python

Navigation

-

Contents:

+

Documentation Content:

diff --git a/docs/build/html/_modules/celpy/evaluation.html b/docs/build/html/_modules/celpy/evaluation.html index 98fba57..cde0de7 100644 --- a/docs/build/html/_modules/celpy/evaluation.html +++ b/docs/build/html/_modules/celpy/evaluation.html @@ -47,30 +47,38 @@

Source code for celpy.evaluation

 # See the License for the specific language governing permissions and limitations under the License.
 
 """
-CEL Interpreter using the AST directly.
+Evaluates CEL expressions given an AST.
+
+There are two implementations:
+
+-   Evaluator -- interprets the AST directly.
+
+-   Transpiler -- transpiles the AST to Python, compiles the Python to create a code object, and then uses :py:func:`exec` to evaluate the code object.
 
 The general idea is to map CEL operators to Python operators and push the
 real work off to Python objects defined by the :py:mod:`celpy.celtypes` module.
 
-CEL operator "+" is implemented by "_+_" function. We map this to :py:func:`operator.add`.
-This will then look for `__add__()` methods in the various :py:class:`celpy.celtypes.CELType`
+CEL operator ``+`` is implemented by a ``"_+_"`` function.
+We map this name to :py:func:`operator.add`.
+This will then look for :py:meth:`__add__` methods in the various :py:mod:`celpy.celtypes`
 types.
 
 In order to deal gracefully with missing and incomplete data,
-exceptions are turned into first-class :py:class:`Result` objects.
+checked exceptions are used.
+A raised exception is turned into first-class :py:class:`celpy.celtypes.Result` object.
 They're not raised directly, but instead saved as part of the evaluation so that
 short-circuit operators can ignore the exceptions.
 
 This means that Python exceptions like :exc:`TypeError`, :exc:`IndexError`, and :exc:`KeyError`
 are caught and transformed into :exc:`CELEvalError` objects.
 
-The :py:class:`Resut` type hint is a union of the various values that are encountered
+The :py:class:`celpy.celtypes.Result` type hint is a union of the various values that are encountered
 during evaluation. It's a union of the :py:class:`celpy.celtypes.CELTypes` type and the
 :exc:`CELEvalError` exception.
 
 ..  important:: Debugging
 
-    If the os environment variable ``CEL_TRACE`` is set, then detailed tracing of methods is made available.
+    If the OS environment variable :envvar:`CEL_TRACE` is set, then detailed tracing of methods is made available.
     To see the trace, set the logging level for ``celpy.Evaluator`` to ``logging.DEBUG``.
 
 """
@@ -80,8 +88,10 @@ 

Source code for celpy.evaluation

 import operator
 import os
 import re
+from string import Template
 import sys
 from functools import reduce, wraps
+from textwrap import dedent
 from typing import (
     Any,
     Callable,
@@ -124,7 +134,7 @@ 

Source code for celpy.evaluation

 except ImportError:  # pragma: no cover
 
 
-[docs] +[docs] def function_matches(text: str, pattern: str) -> "Result": try: m = re.search(pattern, text) @@ -135,9 +145,9 @@

Source code for celpy.evaluation

 
 
 
-# This annotation describes a union of types, functions, and function types.
+# An Annotation describes a union of types, functions, and function types.
 Annotation = Union[
-    celpy.celtypes.CELType,
+    celpy.celtypes.TypeType,
     Callable[
         ..., celpy.celtypes.Value
     ],  # Conversion functions and protobuf message type
@@ -148,12 +158,12 @@ 

Source code for celpy.evaluation

 
 
 
-[docs] +[docs] class CELSyntaxError(Exception): """CEL Syntax error -- the AST did not have the expected structure."""
-[docs] +[docs] def __init__( self, arg: Any, line: Optional[int] = None, column: Optional[int] = None ) -> None: @@ -165,12 +175,12 @@

Source code for celpy.evaluation

 
 
 
-[docs] +[docs] class CELUnsupportedError(Exception): """Feature unsupported by this implementation of CEL."""
-[docs] +[docs] def __init__(self, arg: Any, line: int, column: int) -> None: super().__init__(arg) self.line = line @@ -180,7 +190,7 @@

Source code for celpy.evaluation

 
 
 
-[docs] +[docs] class CELEvalError(Exception): """CEL evaluation problem. This can be saved as a temporary value for later use. This is politely ignored by logic operators to provide commutative short-circuit. @@ -190,7 +200,7 @@

Source code for celpy.evaluation

     """
 
 
-[docs] +[docs] def __init__( self, *args: Any, @@ -211,7 +221,7 @@

Source code for celpy.evaluation

 
 
 
-[docs] +[docs] def __repr__(self) -> str: cls = self.__class__.__name__ if self.tree and self.token: @@ -225,103 +235,103 @@

Source code for celpy.evaluation

 
 
 
-[docs] +[docs] def with_traceback(self, tb: Any) -> "CELEvalError": return super().with_traceback(tb)
-[docs] +[docs] def __neg__(self) -> "CELEvalError": return self
-[docs] +[docs] def __add__(self, other: Any) -> "CELEvalError": return self
-[docs] +[docs] def __sub__(self, other: Any) -> "CELEvalError": return self
-[docs] +[docs] def __mul__(self, other: Any) -> "CELEvalError": return self
-[docs] +[docs] def __truediv__(self, other: Any) -> "CELEvalError": return self
-[docs] +[docs] def __floordiv__(self, other: Any) -> "CELEvalError": return self
-[docs] +[docs] def __mod__(self, other: Any) -> "CELEvalError": return self
-[docs] +[docs] def __pow__(self, other: Any) -> "CELEvalError": return self
-[docs] +[docs] def __radd__(self, other: Any) -> "CELEvalError": return self
-[docs] +[docs] def __rsub__(self, other: Any) -> "CELEvalError": return self
-[docs] +[docs] def __rmul__(self, other: Any) -> "CELEvalError": return self
-[docs] +[docs] def __rtruediv__(self, other: Any) -> "CELEvalError": return self
-[docs] +[docs] def __rfloordiv__(self, other: Any) -> "CELEvalError": return self
-[docs] +[docs] def __rmod__(self, other: Any) -> "CELEvalError": return self
-[docs] +[docs] def __rpow__(self, other: Any) -> "CELEvalError": return self
-[docs] +[docs] def __eq__(self, other: Any) -> bool: if isinstance(other, CELEvalError): return self.args == other.args @@ -329,14 +339,14 @@

Source code for celpy.evaluation

 
 
 
-[docs] +[docs] def __call__(self, *args: Any) -> "CELEvalError": return self
-# The interim results extend ``celtypes`` to include itermediate ``CELEvalError`` exception objects. +# The interim results extend ``celtypes`` to include intermediate ``CELEvalError`` exception objects. # These can be deferred as part of commutative logical_and and logical_or operations. # It includes the responses to ``type()`` queries, also. Result = Union[ @@ -362,7 +372,7 @@

Source code for celpy.evaluation

 
 
 
-[docs] +[docs] def eval_error( new_text: str, exc_class: Exception_Filter ) -> Callable[[TargetFunc], TargetFunc]: @@ -370,7 +380,7 @@

Source code for celpy.evaluation

     Wrap a function to transform native Python exceptions to CEL CELEvalError values.
     Any exception of the given class is replaced with the new CELEvalError object.
 
-    :param new_text: Text of the exception, e.g., "divide by zero", "no such overload")
+    :param new_text: Text of the exception, e.g., "divide by zero", "no such overload",
         this is the return value if the :exc:`CELEvalError` becomes the result.
     :param exc_class: A Python exception class to match, e.g. ZeroDivisionError,
         or a sequence of exception classes (e.g. (ZeroDivisionError, ValueError))
@@ -406,12 +416,12 @@ 

Source code for celpy.evaluation

 
 
 
-[docs] +[docs] def boolean( function: Callable[..., celpy.celtypes.Value], ) -> Callable[..., celpy.celtypes.BoolType]: """ - Wraps boolean operators to create CEL BoolType results. + Wraps operators to create CEL BoolType results. :param function: One of the operator.lt, operator.gt, etc. comparison functions :return: Decorated function with type coercion. @@ -421,17 +431,17 @@

Source code for celpy.evaluation

     def bool_function(
         a: celpy.celtypes.Value, b: celpy.celtypes.Value
     ) -> celpy.celtypes.BoolType:
-        result = function(a, b)
-        if result == NotImplemented:
-            return cast(celpy.celtypes.BoolType, result)
-        return celpy.celtypes.BoolType(bool(result))
+        result_value = function(a, b)
+        if result_value == NotImplemented:
+            return cast(celpy.celtypes.BoolType, result_value)
+        return celpy.celtypes.BoolType(bool(result_value))
 
     return bool_function
-[docs] +[docs] def operator_in(item: Result, container: Result) -> Result: """ CEL contains test; ignores type errors. @@ -452,43 +462,44 @@

Source code for celpy.evaluation

     would do it. But. It's not quite right for the job.
 
     There need to be three results, something :py:func:`filter` doesn't handle.
-    These are the chocies:
+    These are the choices:
 
     -   True. There was a item found. Exceptions may or may not have been found.
-    -   False. No item found AND no expceptions.
+    -   False. No item found AND no exceptions.
     -   CELEvalError. No item found AND at least one exception.
 
     To an extent this is a little like the ``exists()`` macro.
     We can think of ``container.contains(item)`` as ``container.exists(r, r == item)``.
-    However, exists() tends to silence exceptions, where this can expost them.
+    However, exists() tends to silence exceptions, where this can expose them.
 
     ..  todo:: This may be better done as
 
         ``reduce(logical_or, (item == c for c in container), BoolType(False))``
     """
-    result: Result = celpy.celtypes.BoolType(False)
+    result_value: Result = celpy.celtypes.BoolType(False)
     for c in cast(Iterable[Result], container):
         try:
             if c == item:
                 return celpy.celtypes.BoolType(True)
         except TypeError as ex:
             logger.debug("operator_in(%s, %s) --> %s", item, container, ex)
-            result = CELEvalError("no such overload", ex.__class__, ex.args)
-    logger.debug("operator_in(%r, %r) = %r", item, container, result)
-    return result
+ result_value = CELEvalError("no such overload", ex.__class__, ex.args) + logger.debug("operator_in(%r, %r) = %r", item, container, result_value) + return result_value
-[docs] +[docs] def function_size(container: Result) -> Result: """ - The size() function applied to a Value. Delegate to Python's :py:func:`len`. + The size() function applied to a Value. + This is delegated to Python's :py:func:`len`. - (string) -> int string length - (bytes) -> int bytes length - (list(A)) -> int list size - (map(A, B)) -> int map size + size(string) -> int string length + size(bytes) -> int bytes length + size(list(A)) -> int list size + size(map(A, B)) -> int map size For other types, this will raise a Python :exc:`TypeError`. (This is captured and becomes an :exc:`CELEvalError` Result.) @@ -499,14 +510,190 @@

Source code for celpy.evaluation

     if container is None:
         return celpy.celtypes.IntType(0)
     sized_container = cast(Sized, container)
-    result = celpy.celtypes.IntType(len(sized_container))
-    logger.debug("function_size(%r) = %r", container, result)
-    return result
+ result_value = celpy.celtypes.IntType(len(sized_container)) + logger.debug("function_size(%r) = %r", container, result_value) + return result_value
+ + + +
+[docs] +def function_contains( + container: Union[ + celpy.celtypes.ListType, celpy.celtypes.MapType, celpy.celtypes.StringType + ], + item: Result, +) -> Result: + """ + The contains() function applied to a Container and a Value. + THis is delegated to the `contains` method of a class. + """ + return celpy.celtypes.BoolType(container.contains(cast(celpy.celtypes.Value, item)))
+ + + +
+[docs] +def function_startsWith( + string: celpy.celtypes.StringType, fragment: celpy.celtypes.StringType +) -> Result: + return celpy.celtypes.BoolType(string.startswith(fragment))
+ + + +
+[docs] +def function_endsWith( + string: celpy.celtypes.StringType, fragment: celpy.celtypes.StringType +) -> Result: + return celpy.celtypes.BoolType(string.endswith(fragment))
+ + + +
+[docs] +def function_getDate( + ts: celpy.celtypes.TimestampType, + tz_name: Optional[celpy.celtypes.StringType] = None, +) -> Result: + return celpy.celtypes.IntType(ts.getDate(tz_name))
+ + + +
+[docs] +def function_getDayOfMonth( + ts: celpy.celtypes.TimestampType, + tz_name: Optional[celpy.celtypes.StringType] = None, +) -> Result: + return celpy.celtypes.IntType(ts.getDayOfMonth(tz_name))
+ + + +
+[docs] +def function_getDayOfWeek( + ts: celpy.celtypes.TimestampType, + tz_name: Optional[celpy.celtypes.StringType] = None, +) -> Result: + return celpy.celtypes.IntType(ts.getDayOfWeek(tz_name))
+ + + +
+[docs] +def function_getDayOfYear( + ts: celpy.celtypes.TimestampType, + tz_name: Optional[celpy.celtypes.StringType] = None, +) -> Result: + return celpy.celtypes.IntType(ts.getDayOfYear(tz_name))
+ + + +
+[docs] +def function_getFullYear( + ts: celpy.celtypes.TimestampType, + tz_name: Optional[celpy.celtypes.StringType] = None, +) -> Result: + return celpy.celtypes.IntType(ts.getFullYear(tz_name))
+ + + +
+[docs] +def function_getMonth( + ts: celpy.celtypes.TimestampType, + tz_name: Optional[celpy.celtypes.StringType] = None, +) -> Result: + return celpy.celtypes.IntType(ts.getMonth(tz_name))
+ + + +
+[docs] +def function_getHours( + ts: celpy.celtypes.TimestampType, + tz_name: Optional[celpy.celtypes.StringType] = None, +) -> Result: + return celpy.celtypes.IntType(ts.getHours(tz_name))
+ + + +
+[docs] +def function_getMilliseconds( + ts: celpy.celtypes.TimestampType, + tz_name: Optional[celpy.celtypes.StringType] = None, +) -> Result: + return celpy.celtypes.IntType(ts.getMilliseconds(tz_name))
+ + + +
+[docs] +def function_getMinutes( + ts: celpy.celtypes.TimestampType, + tz_name: Optional[celpy.celtypes.StringType] = None, +) -> Result: + return celpy.celtypes.IntType(ts.getMinutes(tz_name))
+ + + +
+[docs] +def function_getSeconds( + ts: celpy.celtypes.TimestampType, + tz_name: Optional[celpy.celtypes.StringType] = None, +) -> Result: + return celpy.celtypes.IntType(ts.getSeconds(tz_name))
+ + + +
+[docs] +def bool_lt(a: Result, b: Result) -> Result: + return boolean(operator.lt)(a, b)
+ + + +
+[docs] +def bool_le(a: Result, b: Result) -> Result: + return boolean(operator.le)(a, b)
+ + + +
+[docs] +def bool_gt(a: Result, b: Result) -> Result: + return boolean(operator.gt)(a, b)
+ + + +
+[docs] +def bool_ge(a: Result, b: Result) -> Result: + return boolean(operator.ge)(a, b)
+ + + +
+[docs] +def bool_eq(a: Result, b: Result) -> Result: + return boolean(operator.eq)(a, b)
+ + + +
+[docs] +def bool_ne(a: Result, b: Result) -> Result: + return boolean(operator.ne)(a, b)
# User-defined functions can override items in this mapping. -base_functions: Mapping[str, CELFunction] = { +base_functions: dict[str, CELFunction] = { "!_": celpy.celtypes.logical_not, "-_": operator.neg, "_+_": operator.add, @@ -514,49 +701,42 @@

Source code for celpy.evaluation

     "_*_": operator.mul,
     "_/_": operator.truediv,
     "_%_": operator.mod,
-    "_<_": boolean(operator.lt),
-    "_<=_": boolean(operator.le),
-    "_>=_": boolean(operator.ge),
-    "_>_": boolean(operator.gt),
-    "_==_": boolean(operator.eq),
-    "_!=_": boolean(operator.ne),
+    "_<_": bool_lt,
+    "_<=_": bool_le,
+    "_>_": bool_gt,
+    "_>=_": bool_ge,
+    "_==_": bool_eq,
+    "_!=_": bool_ne,
     "_in_": operator_in,
     "_||_": celpy.celtypes.logical_or,
     "_&&_": celpy.celtypes.logical_and,
     "_?_:_": celpy.celtypes.logical_condition,
     "_[_]": operator.getitem,
+    # The "methods" are actually named functions that can be overridden.
+    # The function version delegates to class methods.
+    # Yes, it's a bunch of indirection, but it permits simple overrides.
+    # A number of types support "size" and "contains": StringType, MapType, ListType
+    # This is generally made available via the _in_ operator.
     "size": function_size,
-    # StringType methods
-    "endsWith": lambda s, text: celpy.celtypes.BoolType(s.endswith(text)),
-    "startsWith": lambda s, text: celpy.celtypes.BoolType(s.startswith(text)),
+    "contains": function_contains,
+    # Universally available
+    "type": celpy.celtypes.TypeType,
+    # StringType methods, used by :py:meth:`Evaluator.method_eval`
+    "endsWith": function_endsWith,
+    "startsWith": function_startsWith,
     "matches": function_matches,
-    "contains": lambda s, text: celpy.celtypes.BoolType(text in s),
     # TimestampType methods. Type details are redundant, but required because of the lambdas
-    "getDate": lambda ts, tz_name=None: celpy.celtypes.IntType(ts.getDate(tz_name)),
-    "getDayOfMonth": lambda ts, tz_name=None: celpy.celtypes.IntType(
-        ts.getDayOfMonth(tz_name)
-    ),
-    "getDayOfWeek": lambda ts, tz_name=None: celpy.celtypes.IntType(
-        ts.getDayOfWeek(tz_name)
-    ),
-    "getDayOfYear": lambda ts, tz_name=None: celpy.celtypes.IntType(
-        ts.getDayOfYear(tz_name)
-    ),
-    "getFullYear": lambda ts, tz_name=None: celpy.celtypes.IntType(
-        ts.getFullYear(tz_name)
-    ),
-    "getMonth": lambda ts, tz_name=None: celpy.celtypes.IntType(ts.getMonth(tz_name)),
+    "getDate": function_getDate,
+    "getDayOfMonth": function_getDayOfMonth,
+    "getDayOfWeek": function_getDayOfWeek,
+    "getDayOfYear": function_getDayOfYear,
+    "getFullYear": function_getFullYear,
+    "getMonth": function_getMonth,
     # TimestampType and DurationType methods
-    "getHours": lambda ts, tz_name=None: celpy.celtypes.IntType(ts.getHours(tz_name)),
-    "getMilliseconds": lambda ts, tz_name=None: celpy.celtypes.IntType(
-        ts.getMilliseconds(tz_name)
-    ),
-    "getMinutes": lambda ts, tz_name=None: celpy.celtypes.IntType(
-        ts.getMinutes(tz_name)
-    ),
-    "getSeconds": lambda ts, tz_name=None: celpy.celtypes.IntType(
-        ts.getSeconds(tz_name)
-    ),
+    "getHours": function_getHours,
+    "getMilliseconds": function_getMilliseconds,
+    "getMinutes": function_getMinutes,
+    "getSeconds": function_getSeconds,
     # type conversion functions
     "bool": celpy.celtypes.BoolType,
     "bytes": celpy.celtypes.BytesType,
@@ -569,39 +749,39 @@ 

Source code for celpy.evaluation

     "string": celpy.celtypes.StringType,
     "timestamp": celpy.celtypes.TimestampType,
     "uint": celpy.celtypes.UintType,
-    "type": type,
 }
 
 
 
-[docs] +[docs] class Referent: """ A Name can refer to any of the following things: - - Annotations -- initially most names are these - or a CELFunction that may implement a type. + - ``Annotation`` -- initially most names are these. + Must be provided as part of the initialization. + + - ``CELFunction`` -- a Python function to implement a CEL function or method. Must be provided as part of the initialization. + The type conversion functions are names in a ``NameContainer``. - - NameContainer -- some names are these. This is true - when the name is *not* provided as part of the initialization because + - ``NameContainer`` -- some names are these. + This is true when the name is *not* provided as part of the initialization because we discovered the name during type or environment binding. - - celpy.celtypes.Value -- many annotations also have values. + - ``celpy.celtypes.Value`` -- many annotations also have values. These are provided **after** Annotations, and require them. - - CELEvalError -- This seems unlikely, but we include it because it's possible. - - - Functions -- All of the type conversion functions are names in a NameContainer. + - ``CELEvalError`` -- This seems unlikely, but we include it because it's possible. - A name can be ambiguous and refer to both a nested ``NameContainer`` as well - as a ``celpy.celtypes.Value`` (usually a MapType instance.) + A name can be ambiguous and refer to a nested ``NameContainer`` as well + as a ``celpy.celtypes.Value`` (usually a ``MapType`` instance.) Object ``b`` has two possible meanings: - - ``b.c`` is a NameContainer for ``c``, a string. + - ``b`` is a ``NameContainer`` with ``c``, a string or some other object. - - ``b`` is a mapping, and ``b.c`` is syntax sugar for ``b['c']``. + - ``b`` is a ``MapType`` or ``MessageType``, and ``b.c`` is syntax sugar for ``b['c']``. The "longest name" rule means that the useful value is the "c" object in the nested ``NameContainer``. @@ -616,26 +796,25 @@

Source code for celpy.evaluation

     >>> b.value == nc
     True
 
-    In effect, this class is
-    ::
+    ..  note:: Future Design
+
+        A ``Referent`` is (almost) a ``tuple[Annotation, NameContainer | None, Value | NotSetSentinel]``.
+        The current implementation is stateful, because values are optional and may be added later.
+        The use of a special sentinel to indicate the value was not set is a little akward.
+        It's not really a 3-tuple, because NameContainers don't have values; they are a kind of value.
+        (``None`` is a valid value, and can't be used for this.)
 
-        Referent = Union[
-            Annotation,
-            celpy.celtypes.Value,
-            CELEvalError,
-            CELFunction,
-        ]
+        It may be slightly simpler to use a union of two types:
+        ``tuple[Annotation] | tuple[Annotation, NameContainer | Value]``.
+        One-tuples capture the Annotation for a name; two-tuples capture Annotation and Value (or subsidiary NameContainer).
     """
 
 
-[docs] +[docs] def __init__( self, ref_to: Optional[Annotation] = None, - # Union[ - # None, Annotation, celpy.celtypes.Value, CELEvalError, - # CELFunction, 'NameContainer' - # ] = None + # TODO: Add value here, also, as a handy short-cut to avoid the value setter. ) -> None: self.annotation: Optional[Annotation] = None self.container: Optional["NameContainer"] = None @@ -647,13 +826,13 @@

Source code for celpy.evaluation

             CELFunction,
             "NameContainer",
         ] = None
-        self._value_set = False
+        self._value_set = False  # Should NOT be private.
         if ref_to:
             self.annotation = ref_to
-[docs] +[docs] def __repr__(self) -> str: return ( f"{self.__class__.__name__}(annotation={self.annotation!r}, " @@ -662,6 +841,21 @@

Source code for celpy.evaluation

         )
+
+[docs] + def __eq__(self, other: Any) -> bool: + # TODO: When minimum version >= 3.10, use match statement + if isinstance(other, type(self)): + same = ( + self.annotation == other.annotation + and self.container == other.container + and self._value_set == other._value_set + and (self._value == other._value if self._value_set else True) + ) + return same + return NotImplemented # pragma: no cover
+ + @property def value( self, @@ -693,7 +887,7 @@

Source code for celpy.evaluation

         self._value_set = True
 
 
-[docs] +[docs] def clone(self) -> "Referent": new = Referent(self.annotation) new.container = self.container @@ -704,9 +898,9 @@

Source code for celpy.evaluation

 
 
 
-# A name resolution context is a mapping from an identifer to a Value or a ``NameContainer``.
+# A name resolution context is a mapping from an identifier to a Value or a ``NameContainer``.
 # This reflects some murkiness in the name resolution algorithm that needs to be cleaned up.
-Context = Mapping[str, Union[Result, "NameContainer"]]
+Context = Mapping[str, Union[Result, "NameContainer", "CELFunction"]]
 
 
 # Copied from cel.lark
@@ -714,7 +908,7 @@ 

Source code for celpy.evaluation

 
 
 
-[docs] +[docs] class NameContainer(Dict[str, Referent]): """ A namespace that fulfills the CEL name resolution requirement. @@ -754,33 +948,33 @@

Source code for celpy.evaluation

     The annotations (i.e. protobuf message types) have only a fairly remote annotation without
     a value.
 
-    Structure.
+    ..  rubric:: Structure
 
-    A NameContainer is a mapping from names to Referents.
+    A ``NameContainer`` is a mapping from names to ``Referent`` instances.
 
-    A Referent can be one of three things.
+    A `Referent` can be one of several things, including...
 
     -   A NameContainer further down the path
     -   An Annotation
-    -   An Annotation with a value.
+    -   An Annotation and a value
+    -   A CELFunction (In effect, an Annotation of CELFunction, and a value of the function implementation.)
 
-    Loading Names.
+    ..  rubric:: Life and Content
 
-    There are several "phases" to building the chain of ``NameContainer`` instances.
+    There are two phases to building the chain of ``NameContainer`` instances.
 
     1.  The ``Activation`` creates the initial ``name : annotation`` bindings.
         Generally, the names are type names, like "int", bound to :py:class:`celtypes.IntType`.
         In some cases, the name is a future variable name, "resource",
         bound to :py:class:`celtypes.MapType`.
 
-    2.  The ``Activation`` creates a second ``NameContainer`` that has variable names.
-        This has a reference back to the parent to resolve names that are types.
+    2.  The ``Activation`` updates some variables to provide values.
 
-    This involves decomposing the paths of names to make a tree of nested ``NameContainers``.
+    A name is decomposed into a path to make a tree of nested ``NameContainers``.
     Upper-level containers don't (necessarily) have types or values -- they're merely
     ``NameContainer`` along the path to the target names.
 
-    Resolving Names.
+    ..  rubric:: Resolving Names
 
     See https://github.com/google/cel-spec/blob/master/doc/langdef.md#name-resolution
 
@@ -825,7 +1019,7 @@ 

Source code for celpy.evaluation

 
     The evaluator's :meth:`ident_value` method resolves the identifier into the ``Referent``.
 
-    Acceptance Test Case
+    ..  rubric:: Acceptance Test Cases
 
     We have two names
 
@@ -849,7 +1043,7 @@ 

Source code for celpy.evaluation

     logger = logging.getLogger("celpy.NameContainer")
 
 
-[docs] +[docs] def __init__( self, name: Optional[str] = None, @@ -864,7 +1058,7 @@

Source code for celpy.evaluation

 
 
 
-[docs] +[docs] def load_annotations( self, names: Mapping[str, Annotation], @@ -883,7 +1077,7 @@

Source code for celpy.evaluation

         :param names: A dictionary of {"name1.name1....": Referent, ...} items.
         """
         for name, refers_to in names.items():
-            self.logger.debug("load_annotations %r : %r", name, refers_to)
+            # self.logger.debug("load_annotations %r : %r", name, refers_to)
             if not self.extended_name_path.match(name):
                 raise ValueError(f"Invalid name {name}")
 
@@ -900,11 +1094,11 @@ 

Source code for celpy.evaluation

 
 
 
-[docs] +[docs] def load_values(self, values: Context) -> None: - """Update annotations with actual values.""" + """Update any annotations with actual values.""" for name, refers_to in values.items(): - self.logger.debug("load_values %r : %r", name, refers_to) + # self.logger.debug("load_values %r : %r", name, refers_to) if not self.extended_name_path.match(name): raise ValueError(f"Invalid name {name}") @@ -918,12 +1112,12 @@

Source code for celpy.evaluation

                 if ref.container is None:
                     ref.container = NameContainer(parent=self.parent)
                 context = ref.container
-            context.setdefault(final, Referent())  # No annotation.
+            context.setdefault(final, Referent())  # No annotation previously present.
             context[final].value = refers_to
-[docs] +[docs] class NotFound(Exception): """ Raised locally when a name is not found in the middle of package search. @@ -934,70 +1128,111 @@

Source code for celpy.evaluation

 
 
 
-[docs] +[docs] @staticmethod - def dict_find_name(some_dict: Dict[str, Referent], path: List[str]) -> Result: + def dict_find_name( + some_dict: Union[Dict[str, Referent], Referent], path: Sequence[str] + ) -> Referent: """ - Extension to navgiate into mappings, messages, and packages. + Recursive navigation into mappings, messages, and packages. + These are not NameContainers (or Activations). - :param some_dict: An instance of a MapType, MessageType, or PackageType. - :param path: names to follow into the structure. + :param some_dict: An instance of a ``MapType``, ``MessageType``, or ``PackageType``. + :param path: sequence of names to follow into the structure. :returns: Value found down inside the structure. """ if path: head, *tail = path try: return NameContainer.dict_find_name( - cast(Dict[str, Referent], some_dict[head]), tail + cast(Dict[str, Referent], some_dict)[head], tail ) except KeyError: - NameContainer.logger.debug("%r not found in %s", head, some_dict.keys()) + NameContainer.logger.debug( + "%r not found in %s", + head, + cast(Dict[str, Referent], some_dict).keys(), + ) raise NameContainer.NotFound(path) else: - return cast(Result, some_dict)
+ # End of the path, we found it. + if isinstance(some_dict, Referent): # pragma: no cover + # Seems unlikely, but, just to be sure... + return some_dict + referent = Referent(celpy.celtypes.MapType) + referent.value = cast(celpy.celtypes.MapType, some_dict) + return referent
-[docs] - def find_name(self, path: List[str]) -> Union["NameContainer", Result]: +[docs] + def find_name(self, path: List[str]) -> Referent: """ Find the name by searching down through nested packages or raise NotFound. + Returns the Value associated with this Name. + This is a kind of in-order tree walk of contained packages. + + The collaborator must choose the annotation or the value from the Referent. + + .. todo:: Refactored to return Referent. + + The collaborator must handle two distinct errors: + + 1. ``self[head]`` has a ``KeyError`` exception -- while not found on this path, the collaborator should keep searching. + Eventually it will raise a final ``KeyError`` that maps to a ``CELEvalError`` + This should be exposed as f"no such member in mapping: {ex.args[0]!r}" + + 2. ``self[head].value`` has no value and the ``Referent`` returned the annotation instead of the value. + In transpiled Python code, this **should** be exposed as f"undeclared reference to {ex.args[0]!r} (in container {ex.args[1]!r})" """ - if path: - head, *tail = path - try: - sub_context = self[head].value - except KeyError: - self.logger.debug("%r not found in %s", head, self.keys()) - raise NameContainer.NotFound(path) - if isinstance(sub_context, NameContainer): - return sub_context.find_name(tail) - elif isinstance( - sub_context, - ( - celpy.celtypes.MessageType, - celpy.celtypes.MapType, - celpy.celtypes.PackageType, - dict, - ), - ): - # Out of defined NameContainers, moving into Values: Messages, Mappings or Packages - # Make a fake Referent return value. - item: Union["NameContainer", Result] = NameContainer.dict_find_name( - cast(Dict[str, Referent], sub_context), tail - ) - return item - else: - # Fully matched. No more Referents with NameContainers or Referents with Mappings. - return cast(NameContainer, sub_context) + if not path: + # Already fully matched. This ``NameContainer`` is what they were looking for. + referent = Referent() + referent.value = self + return referent + + # Find the head of the path. + head, *tail = path + try: + sub_context = self[head] + except KeyError: + self.logger.debug("%r not found in %r", head, list(self.keys())) + raise NameContainer.NotFound(path) + if not tail: + # Found what they were looking for + return sub_context + + # There are several special cases for the continued search. + self.logger.debug("%r %r %r", head, tail, sub_context) + + # We found a NameContainer, simple recursion will do. + item: Referent + if sub_context.container: # isinstance(sub_context, NameContainer): + return sub_context.container.find_name(tail) + + # Uncommon case: value with no annotation, and the value is a Message, Mapping, or Package + elif sub_context._value_set and isinstance( + sub_context.value, + ( + celpy.celtypes.MessageType, + celpy.celtypes.MapType, + celpy.celtypes.PackageType, + dict, + ), + ): + item = NameContainer.dict_find_name( + cast(Dict[str, Referent], sub_context.value), tail + ) + return item + + # A primitive type, but not at the end of the path. Ugn. else: - # Fully matched. This NameContainer is what we were looking for. - return self
+ raise TypeError(f"{sub_context!r} not a container")
-[docs] +[docs] def parent_iter(self) -> Iterator["NameContainer"]: """Yield this NameContainer and all of its parents to create a flat list.""" yield self @@ -1006,7 +1241,7 @@

Source code for celpy.evaluation

 
 
 
-[docs] +[docs] def resolve_name(self, package: Optional[str], name: str) -> Referent: """ Search with less and less package prefix until we find the thing. @@ -1015,11 +1250,11 @@

Source code for celpy.evaluation

         If a.b is a name to be resolved in the context of a protobuf declaration with scope A.B,
         then resolution is attempted, in order, as
 
-        1. A.B.a.b.  (Search for "a" in paackage "A.B"; the ".b" is handled separately.)
+        1. A.B.a.b.  (Search for "a" in package "A.B"; the ".b" is handled separately.)
 
-        2. A.a.b.  (Search for "a" in paackage "A"; the ".b" is handled separately.)
+        2. A.a.b.  (Search for "a" in package "A"; the ".b" is handled separately.)
 
-        3. (finally) a.b.  (Search for "a" in paackage None; the ".b" is handled separately.)
+        3. (finally) a.b.  (Search for "a" in package None; the ".b" is handled separately.)
 
         To override this behavior, one can use .a.b;
         this name will only be attempted to be resolved in the root scope, i.e. as a.b.
@@ -1029,22 +1264,23 @@ 

Source code for celpy.evaluation

         Given a target, search through this ``NameContainer`` and all parents in the
         :meth:`parent_iter` iterable.
         The first name we find in the parent sequence is the goal.
-        This is because values are first, type annotations are laast.
+        This is because values are first, type annotations are last.
 
         If we can't find the identifier with given package target,
         truncate the package name from the end to create a new target and try again.
         This is a bottom-up look that favors the longest name.
 
-        :param package: Prefix string "name.name.name"
+        :param package: Prefix string "path.path.path"
         :param name: The variable we're looking for
-        :return: Name resolution as a Rereferent, often a value, but maybe a package or an
+        :return: Name resolution as a ``Rereferent``, often a value, but maybe a package or an
             annotation.
+        :raises KeyError: if the name cannot be found in this ``NameContainer``
         """
         self.logger.debug(
             "resolve_name(%r.%r) in %s, parent=%s",
             package,
             name,
-            self.keys,
+            list(self.keys()),
             self.parent,
         )
         # Longest Name
@@ -1053,31 +1289,34 @@ 

Source code for celpy.evaluation

         else:
             target = [""]
         # Pool of matches
-        matches: List[Tuple[List[str], Union["NameContainer", Result]]] = []
+        matches: List[Tuple[List[str], Referent]] = []
         # Target has an extra item to make the len non-zero.
         while not matches and target:
             target = target[:-1]
-            for p in self.parent_iter():
+            for nc in self.parent_iter():
                 try:
                     package_ident: List[str] = target + [name]
-                    match: Union["NameContainer", Result] = p.find_name(package_ident)
-                    matches.append((package_ident, match))
+                    # Find the Referent for this name.
+                    ref_to = nc.find_name(package_ident)
+                    matches.append((package_ident, ref_to))
                 except NameContainer.NotFound:
                     # No matches; move to the parent and try again.
                     pass
             self.logger.debug(
                 "resolve_name: target=%s+[%r], matches=%s", target, name, matches
             )
+        # NOTE: There are two separate kinds of failures: no name at all, and no value for the name.
+        # This is the no name at all. The collaborator may need the value or the annotation.
         if not matches:
             raise KeyError(name)
+        # Find the longest name match and return the Referent.
         # This feels hackish -- it should be the first referent value.
-        # Find the longest name match.p
-        path, value = max(matches, key=lambda path_value: len(path_value[0]))
-        return cast(Referent, value)
+ path, best_match = max(matches, key=lambda path_value: len(path_value[0])) + return best_match
-[docs] +[docs] def clone(self) -> "NameContainer": new = NameContainer(parent=self.parent) for k, v in self.items(): @@ -1085,8 +1324,25 @@

Source code for celpy.evaluation

         return new
+
+[docs] + def get( # type: ignore[override] + self, name: str, default: Optional[Referent] = None + ) -> Union[ + Annotation, celpy.celtypes.Value, CELEvalError, CELFunction, "NameContainer" + ]: + """ + Used by transpiled code to get values from a NameContainer of Referents. + + .. important:: This does not get a Referent, it gets a value. + + .. todo:: This is a poorly-chosen name; a number of related types **all** need to have a get_value() method. + """ + return self.resolve_name(None, name).value
+ +
-[docs] +[docs] def __repr__(self) -> str: return f"{self.__class__.__name__}({dict(self)}, parent={self.parent})"
@@ -1094,10 +1350,11 @@

Source code for celpy.evaluation

 
 
 
-[docs] +[docs] class Activation: """ Namespace with variable bindings and type name ("annotation") bindings. + Additionally, the pool of functions and types are here, also. .. rubric:: Life and Content @@ -1107,22 +1364,22 @@

Source code for celpy.evaluation

 
     A nested Activation is created each time we evaluate a macro.
 
-    An Activation contains a ``NameContainer`` instance to resolve identifers.
+    An Activation contains a ``NameContainer`` instance to resolve identifiers.
     (This may be a needless distinction and the two classes could, perhaps, be combined.)
 
-    ..  todo:: The environment's annotations are type names used for protobuf.
+    These names include variables as well as type names used for protobuf and the internal CEL ``type()`` function.
 
     ..  rubric:: Chaining/Nesting
 
     Activations can form a chain so locals are checked first.
     Activations can nest via macro evaluation, creating transient local variables.
 
+    Consider this CEL macro expression:
     ::
 
         ``"[2, 4, 6].map(n, n / 2)"``
 
-    means nested activations with ``n`` bound to 2, 4, and 6 respectively.
-    The resulting objects then form a resulting list.
+    This works via a nested activation with ``n`` bound to 2, 4, and 6 respectively.
 
     This is used by an :py:class:`Evaluator` as follows::
 
@@ -1159,95 +1416,122 @@ 

Source code for celpy.evaluation

               "namespace resolution should try to find the longest prefix for the evaluator."
 
     Most names start with ``IDENT``, but a primary can start with ``.``.
+    A leading ``.`` changes the search order from most local first to root first.
     """
 
 
-[docs] +[docs] def __init__( self, + *, # Keyword only, too many things here. annotations: Optional[Mapping[str, Annotation]] = None, - package: Optional[str] = None, vars: Optional[Context] = None, - parent: Optional["Activation"] = None, + functions: Optional[Union[Mapping[str, CELFunction], list[CELFunction]]] = None, + package: Optional[str] = None, + based_on: Optional["Activation"] = None, ) -> None: """ Create an Activation. - The annotations are loaded first. The variables are loaded second, and placed - in front of the annotations in the chain of name resolutions. Values come before - annotations. + The annotations are loaded first. The variables and their values are loaded second, and placed + in front of the annotations in the chain of name resolutions. + + The Evaluator and the Transpiler use this to resolve identifiers into types, values, or functions. - :param annotations: Variables and type annotations. + :keyword annotations: Variables and type annotations. Annotations are loaded first to serve as defaults to create a parent NameContainer. - :param package: The package name to assume as a prefix for name resolution. - :param vars: Variables and their values, loaded to update the NameContainer. - :param parent: A parent activation in the case of macro evaluations. + :keyword vars: Variables and their values, loaded to update the NameContainer. + :keyword functions: functions and their implementation, loaded to update the NameContainer. + :keyword package: The package name to assume as a prefix for name resolution. + :keyword based_on: A foundational activation on which this is based. """ logger.debug( - "Activation(annotations=%r, package=%r, vars=%r, parent=%s)", + "Activation(annotations=%r, vars=%r, functions=%r, package=%r, based_on=%s)", annotations, - package, vars, - parent, + functions, + package, + based_on, ) - # Seed the annotation identifiers for this activation. + # Seed the annotations for identifiers in this activation. self.identifiers: NameContainer = NameContainer( - parent=parent.identifiers if parent else None + parent=based_on.identifiers if based_on else None ) if annotations is not None: self.identifiers.load_annotations(annotations) + if vars is not None: + # Set values from a dictionary of names and values. + self.identifiers.load_values(vars) - # The name of the run-time package -- an assumed prefix for name resolution - self.package = package - - # Create a child NameContainer with variables (if any.) - if vars is None: - pass - elif isinstance(vars, Activation): # pragma: no cover - # Deprecated legacy feature. - raise NotImplementedError("Use Activation.clone()") - + # Update this NameContainer functions (if any.) + self.functions: collections.ChainMap[str, CELFunction] + if isinstance(functions, Sequence): + local_functions: dict[str, CELFunction] = { + f.__name__: f for f in functions or [] + } + self.functions = collections.ChainMap(local_functions, base_functions) + # self.identifiers.load_values(local_functions) + elif isinstance(functions, Mapping): + self.functions = collections.ChainMap( + cast(dict[str, CELFunction], functions), base_functions + ) + elif functions is None: + self.functions = collections.ChainMap(base_functions) else: - # Set values from a dictionary of names and values. - self.identifiers.load_values(vars)
+ raise ValueError("functions not a mapping or sequence") # pragma: no cover + + # The name of the run-time package -- an assumed prefix for name resolution + self.package = package
-[docs] +[docs] def clone(self) -> "Activation": """ Create a clone of this activation with a deep copy of the identifiers. """ + logger.debug("Cloning an Activation...") clone = Activation() - clone.package = self.package clone.identifiers = self.identifiers.clone() + clone.functions = self.functions.copy() + clone.package = self.package + logger.debug("clone: %r", self) return clone
-[docs] +[docs] def nested_activation( self, - annotations: Optional[Mapping[str, Annotation]] = None, + annotations: Optional[Mapping[str, Annotation]] = None, # Remove this. vars: Optional[Context] = None, ) -> "Activation": """ - Create a nested sub-Activation that chains to the current activation. - The sub-activations don't have the same implied package context, + Create an Activation based on the current activation. + This new Activation will be seeded from the current activation's + ``NameContainer``. - :param annotations: Variable type annotations - :param vars: Variables with literals to be converted to the desired types. - :return: An ``Activation`` that chains to this Activation. + :param annotations: Optional type definitions for the new local variables. + :param vars: Local variables to be added when creating this activation. + :return: A subsidiary ``Activation`` that chains to this Activation. """ - new = Activation( - annotations=annotations, vars=vars, parent=self, package=self.package + logger.debug("Creating nested Activation...") + nested = Activation( + annotations=annotations, # Replace with self.annotations. + vars=vars, + functions=self.functions, + package=self.package, + based_on=self, ) - return new
+ logger.debug("nested: %r", self) + return nested
-[docs] - def resolve_variable(self, name: str) -> Union[Result, NameContainer]: +[docs] + def resolve_variable( + self, name: str + ) -> Union[celpy.celtypes.Value, CELFunction, NameContainer]: """Find the object referred to by the name. An Activation usually has a chain of NameContainers to be searched. @@ -1255,62 +1539,80 @@

Source code for celpy.evaluation

         A variable can refer to an annotation and/or a value and/or a nested
         container.  Most of the time, we want the `value` attribute of the Referent.
         This can be a Result (a Union[Value, CelType])
+
+        There's a subtle difference between a variable without an annotation,
+        and a variable with an annotation, but without a value.
+        """
+        # Will be a Referent. Get Value or Type -- interpreter works with either.
+        logger.debug("resolve_variable(%r)", name)
+        try:
+            referent = self.identifiers.resolve_name(self.package, name)
+            return cast(Union[Result, NameContainer], referent.value)
+        except KeyError:
+            return self.functions[name]
+ + +
+[docs] + def resolve_function( + self, name: str + ) -> Union[CELFunction, celpy.celtypes.TypeType]: + """A short-cut to find functions without looking at Variables first.""" + logger.debug("resolve_function(%r)", name) + return self.functions[name]
+ + +
+[docs] + def __getattr__( + self, name: str + ) -> Union[celpy.celtypes.Value, CELFunction, NameContainer]: + """Handle ``activation.name`` in transpiled code (or ``activation.get('name')``). + + If the name is not in the Activation with a value, a ``NameError`` exception must be raised. + + Note that :py:meth:`Activation.resolve_variable` depends on :py:meth:`NameContainer.find_name`. + The :py:meth:`NameContainer.find_name` method **also** find the value. + + This is -- perhaps -- less than optimal because it can mask the no value set case. """ - container_or_value = self.identifiers.resolve_name(self.package, str(name)) - return cast(Union[Result, NameContainer], container_or_value)
+ # Will be a Referent. Get Value if it was set or raise error if no value set. + try: + referent = self.identifiers.resolve_name(self.package, name) + logger.debug("get/__getattr__(%r) ==> %r", name, referent) + if referent._value_set: + return cast(Union[Result, NameContainer], referent.value) + else: + if referent.container: + return referent.container + elif referent.annotation: + return cast(Union[Result, NameContainer], referent.annotation) + else: + raise RuntimeError(f"Corrupt {self!r}") # pragma: no cover + except KeyError: + logger.debug("get/__getattr__(%r) fallback to functions", name) + return self.functions[name]
+ get = __getattr__ +
-[docs] +[docs] def __repr__(self) -> str: return ( f"{self.__class__.__name__}" f"(annotations={self.identifiers.parent!r}, " f"package={self.package!r}, " f"vars={self.identifiers!r}, " + f"functions={self.functions!r}, " f"parent={self.identifiers.parent})" )
-
-[docs] -class FindIdent(lark.visitors.Visitor_Recursive): - """Locate the ident token at the bottom of an AST. - - This is needed to find the bind variable for macros. - - It works by doing a "visit" on the entire tree, but saving - the details of the ``ident`` nodes only. - """ - -
-[docs] - def __init__(self) -> None: - self.ident_token: Optional[str] = None
- - -
-[docs] - def ident(self, tree: lark.Tree) -> None: - ident_token = cast(lark.Token, tree.children[0]) - self.ident_token = ident_token.value
- - -
-[docs] - @classmethod - def in_tree(cls: Type["FindIdent"], tree: lark.Tree) -> Optional[str]: - fi = FindIdent() - fi.visit(tree) - return fi.ident_token
-
- - -
-[docs] +[docs] def trace( method: Callable[["Evaluator", lark.Tree], Any], ) -> Callable[["Evaluator", lark.Tree], Any]: @@ -1324,9 +1626,9 @@

Source code for celpy.evaluation

     @wraps(method)
     def concrete_method(self: "Evaluator", tree: lark.Tree) -> Any:
         self.logger.debug("%s%r", self.level * "| ", tree)
-        result = method(self, tree)
-        self.logger.debug("%s%s -> %r", self.level * "| ", tree.data, result)
-        return result
+        result_value = method(self, tree)
+        self.logger.debug("%s%s -> %r", self.level * "| ", tree.data, result_value)
+        return result_value
 
     if os.environ.get("CEL_TRACE"):
         return concrete_method
@@ -1336,7 +1638,7 @@ 

Source code for celpy.evaluation

 
 
 
-[docs] +[docs] class Evaluator(lark.visitors.Interpreter[Result]): """ Evaluate an AST in the context of a specific Activation. @@ -1348,45 +1650,48 @@

Source code for celpy.evaluation

     An AST node must call ``self.visit_children(tree)`` explicitly
     to build the values for all the children of this node.
 
-    Exceptions.
+    ..  rubric:: Exceptions
 
     To handle ``2 / 0 || true``, the ``||``, ``&&``, and ``?:`` operators
     do not trivially evaluate and raise exceptions. They bottle up the
     exceptions and treat them as a kind of undecided value.
 
-    Identifiers.
+    ..  rubric:: Identifiers
 
     Identifiers have three meanings:
 
     -   An object. This is either a variable provided in the activation or a function provided
         when building an execution. Objects also have type annotations.
 
-    -   A type annotation without an object, This is used to build protobuf messages.
+    -   A type annotation without an object. This is used to build protobuf messages.
 
     -   A macro name. The ``member_dot_arg`` construct may have a macro.
         Plus the ``ident_arg`` construct may also have a ``dyn()`` or ``has()`` macro.
         See below for more.
 
-    Other than macros, a name maps to an ``Referent`` instance. This will have an
-    annotation and -- perhaps -- an associated object.
+    Other than macros, a name maps to an ``Referent`` instance.
+    This will have an annotation and -- perhaps -- an associated object.
 
     Names have nested paths. ``a.b.c`` is a mapping, ``a``, that contains a mapping, ``b``,
     that contains ``c``.
 
-    **MACROS ARE SPECIAL**.
+    ..  important MACROS ARE SPECIAL
+
+        They aren't simple functions.
+
+        The macros do not **all** simply visit their children to perform evaluation.
 
-    The macros do not **all** simply visit their children to perform evaluation.
     There are three cases:
 
     - ``dyn()`` does effectively nothing.
-      It visits it's children, but also provides progressive type resolution
+      It visits its children, but also provides progressive type resolution
       through annotation of the AST.
 
     - ``has()`` attempts to visit the child and does a boolean transformation
       on the result.
       This is a macro because it doesn't raise an exception for a missing
       member item reference, but instead maps an exception to False.
-      It doesn't return the value found, for a member item reference; instead, it maps
+      It doesn't return the value found for a member item reference; instead, it maps
       this to True.
 
     - The various ``member.macro()`` constructs do **NOT** visit children.
@@ -1403,12 +1708,12 @@ 

Source code for celpy.evaluation

     logger = logging.getLogger("celpy.Evaluator")
 
 
-[docs] +[docs] def __init__( self, ast: lark.Tree, activation: Activation, - functions: Union[Sequence[CELFunction], Mapping[str, CELFunction], None] = None, + # functions: Union[Sequence[CELFunction], Mapping[str, CELFunction], None] = None, # Refactor into Activation ) -> None: """ Create an evaluator for an AST with specific variables and functions. @@ -1416,45 +1721,38 @@

Source code for celpy.evaluation

         :param ast: The AST to evaluate.
         :param activation: The variable bindings to use.
         :param functions: The functions to use. If nothing is supplied, the default
-            global `base_functions` are used. Otherwise a ChainMap is created so
+            global `base_functions` are used. Otherwise, a ``ChainMap`` is created so
             these local functions override the base functions.
         """
         self.ast = ast
         self.base_activation = activation
         self.activation = self.base_activation
-        self.functions: Mapping[str, CELFunction]
-        if isinstance(functions, Sequence):
-            local_functions = {f.__name__: f for f in functions or []}
-            self.functions = collections.ChainMap(local_functions, base_functions)  # type: ignore [arg-type]
-        elif isinstance(functions, Mapping):
-            self.functions = collections.ChainMap(functions, base_functions)  # type: ignore [arg-type]
-        else:
-            self.functions = base_functions
 
         self.level = 0
-        self.logger.debug("activation: %r", self.activation)
-        self.logger.debug("functions: %r", self.functions)
+ self.logger.debug("Evaluator activation: %r", self.activation)
+ # self.logger.debug("functions: %r", self.functions) # Refactor ``self.functions`` into an Activation
-[docs] +[docs] def sub_evaluator(self, ast: lark.Tree) -> "Evaluator": """ Build an evaluator for a sub-expression in a macro. + :param ast: The AST for the expression in the macro. :return: A new `Evaluator` instance. """ - return Evaluator(ast, activation=self.activation, functions=self.functions)
+ return Evaluator(ast, activation=self.activation)
-[docs] +[docs] def set_activation(self, values: Context) -> "Evaluator": """ - Chain a new activation using the given Context. + Create a new activation using the given Context. This is used for two things: - 1. Bind external variables like command-line arguments or environment variables. + 1. Bind external variables. Examples are command-line arguments and environment variables. 2. Build local variable(s) for macro evaluation. """ @@ -1465,7 +1763,7 @@

Source code for celpy.evaluation

 
 
 
-[docs] +[docs] def ident_value(self, name: str, root_scope: bool = False) -> Result_Function: """Resolve names in the current activation. This includes variables, functions, the type registry for conversions, @@ -1473,16 +1771,17 @@

Source code for celpy.evaluation

 
         We may be limited to root scope, which prevents searching through alternative
         protobuf package definitions.
+        In principle, this changes the order of the search.
         """
-        try:
-            return cast(Result, self.activation.resolve_variable(name))
-        except KeyError:
-            return self.functions[name]
+ # try: + return cast(Result, self.activation.resolve_variable(name))
+ # except KeyError: + # return self.functions[name] # Refactor ``self.functions`` into an Activation
-[docs] - def evaluate(self) -> celpy.celtypes.Value: +[docs] + def evaluate(self, context: Optional[Context] = None) -> celpy.celtypes.Value: """ Evaluate this AST and return the value or raise an exception. @@ -1490,9 +1789,11 @@

Source code for celpy.evaluation

 
         -   External clients want the value or the exception.
 
-        -   Internally, we sometimes want to silence CELEvalError exceptions so that
+        -   Internally, we sometimes want to silence ``CELEvalError`` exceptions so that
             we can apply short-circuit logic and choose a non-exceptional result.
         """
+        if context:
+            self.set_activation(context)
         value = self.visit(self.ast)
         if isinstance(value, CELEvalError):
             raise value
@@ -1500,17 +1801,17 @@ 

Source code for celpy.evaluation

 
 
 
-[docs] +[docs] def visit_children(self, tree: lark.Tree) -> List[Result]: """Extend the superclass to track nesting and current evaluation context.""" self.level += 1 - result = super().visit_children(tree) + result_value = super().visit_children(tree) self.level -= 1 - return result
+ return result_value
-[docs] +[docs] def function_eval( self, name_token: lark.Token, exprlist: Optional[Iterable[Result]] = None ) -> Result: @@ -1518,13 +1819,14 @@

Source code for celpy.evaluation

         Function evaluation.
 
         - Object creation and type conversions.
-        - Other built-in functions like size()
+        - Other functions like ``size()`` or ``type()``
         - Extension functions
         """
         function: CELFunction
         try:
             # TODO: Transitive Lookup of function in all parent activation contexts.
-            function = self.functions[name_token.value]
+            # function = self.functions[name_token.value]  # Refactor ``self.functions`` into an Activation
+            function = self.activation.resolve_function(name_token.value)
         except KeyError as ex:
             err = (
                 f"undeclared reference to '{name_token}' "
@@ -1556,7 +1858,7 @@ 

Source code for celpy.evaluation

 
 
 
-[docs] +[docs] def method_eval( self, object: Result, @@ -1564,18 +1866,23 @@

Source code for celpy.evaluation

         exprlist: Optional[Iterable[Result]] = None,
     ) -> Result:
         """
-        Method evaluation. While are (nominally) attached to an object, the only thing
-        actually special is that the object is the first parameter to a function.
+        Method evaluation. While these are (nominally) attached to an object,
+        that would make overrides complicated.
+        Instead, these are functions (which can be overridden).
+        The object must the first parameter to a function.
         """
         function: CELFunction
         try:
             # TODO: Transitive Lookup of function in all parent activation contexts.
-            function = self.functions[method_ident.value]
+            # function = self.functions[method_ident.value]  # Refactor ``self.functions`` into an Activation
+            function = self.activation.resolve_function(method_ident.value)
         except KeyError as ex:
             self.logger.debug(
                 "method_eval(%r, %r, %s) --> %r", object, method_ident, exprlist, ex
             )
-            self.logger.debug("functions: %s", self.functions)
+            self.logger.debug(
+                "functions: %s", self.activation.functions
+            )  # Refactor ``self.functions`` into an Activation
             err = (
                 f"undeclared reference to {method_ident.value!r} "
                 f"(in activation '{self.activation}')"
@@ -1610,7 +1917,7 @@ 

Source code for celpy.evaluation

 
 
 
-[docs] +[docs] def macro_has_eval(self, exprlist: lark.Tree) -> celpy.celtypes.BoolType: """ The has(e.f) macro. @@ -1628,14 +1935,14 @@

Source code for celpy.evaluation

             - If f is a repeated field or map field, has(e.f) indicates whether the field is
               non-empty.
 
-            - If f is a singular or oneof field, has(e.f) indicates whether the field is set.
+            - If f is a singular or "oneof" field, has(e.f) indicates whether the field is set.
 
         4.  If e evaluates to a protocol buffers version 3 message and f is a defined field:
 
             - If f is a repeated field or map field, has(e.f) indicates whether the field is
               non-empty.
 
-            - If f is a oneof or singular message field, has(e.f) indicates whether the field
+            - If f is a "oneof" field or singular message field, has(e.f) indicates whether the field
               is set.
 
             - If f is some other singular field, has(e.f) indicates whether the field's value
@@ -1650,7 +1957,7 @@ 

Source code for celpy.evaluation

 
 
 
-[docs] +[docs] @trace def expr(self, tree: lark.Tree) -> Result: """ @@ -1658,7 +1965,7 @@

Source code for celpy.evaluation

 
         The default implementation short-circuits
         and can ignore a CELEvalError in the two alternative sub-expressions.
-        The conditional sub-expression CELEvalError is propogated out as the result.
+        The conditional sub-expression CELEvalError is propagated out as the result.
 
         See https://github.com/google/cel-spec/blob/master/doc/langdef.md#logical-operators
 
@@ -1673,7 +1980,8 @@ 

Source code for celpy.evaluation

             return values[0]
         elif len(tree.children) == 3:
             # full conditionalor "?" conditionalor ":" expr.
-            func = self.functions["_?_:_"]
+            # func = self.functions["_?_:_"]    # Refactor ``self.functions`` into an Activation
+            func = self.activation.resolve_function("_?_:_")
             cond_value = self.visit(cast(lark.Tree, tree.children[0]))
             left = right = cast(Result, celpy.celtypes.BoolType(False))
             try:
@@ -1700,7 +2008,7 @@ 

Source code for celpy.evaluation

 
 
 
-[docs] +[docs] @trace def conditionalor(self, tree: lark.Tree) -> Result: """ @@ -1714,7 +2022,8 @@

Source code for celpy.evaluation

             values = self.visit_children(tree)
             return values[0]
         elif len(tree.children) == 2:
-            func = self.functions["_||_"]
+            # func = self.functions["_||_"]   # Refactor ``self.functions`` into an Activation
+            func = self.activation.resolve_function("_||_")
             left, right = cast(Tuple[Result, Result], self.visit_children(tree))
             try:
                 return func(left, right)
@@ -1736,7 +2045,7 @@ 

Source code for celpy.evaluation

 
 
 
-[docs] +[docs] @trace def conditionaland(self, tree: lark.Tree) -> Result: """ @@ -1750,7 +2059,8 @@

Source code for celpy.evaluation

             values = self.visit_children(tree)
             return values[0]
         elif len(tree.children) == 2:
-            func = self.functions["_&&_"]
+            # func = self.functions["_&&_"]    # Refactor ``self.functions`` into an Activation
+            func = self.activation.resolve_function("_&&_")
             left, right = cast(Tuple[Result, Result], self.visit_children(tree))
             try:
                 return func(left, right)
@@ -1772,7 +2082,7 @@ 

Source code for celpy.evaluation

 
 
 
-[docs] +[docs] @trace def relation(self, tree: lark.Tree) -> Result: """ @@ -1793,7 +2103,7 @@

Source code for celpy.evaluation

 
             values = self.visit_children(tree)
             func = functions[op_name_map[tree.data]]
-            result = func(*values)
+            result_value = func(*values)
 
         The AST doesn't provide a flat list of values, however.
         """
@@ -1804,6 +2114,7 @@ 

Source code for celpy.evaluation

 
         elif len(tree.children) == 2:
             left_op, right_tree = cast(Tuple[lark.Tree, lark.Tree], tree.children)
+            # Map a node data in parse tree to an operation function.
             op_name = {
                 "relation_lt": "_<_",
                 "relation_le": "_<=_",
@@ -1813,7 +2124,8 @@ 

Source code for celpy.evaluation

                 "relation_ne": "_!=_",
                 "relation_in": "_in_",
             }[left_op.data]
-            func = self.functions[op_name]
+            # func = self.functions[op_name]    # Refactor ``self.functions`` into an Activation
+            func = self.activation.resolve_function(op_name)
             # NOTE: values have the structure [[left], right]
             (left, *_), right = cast(
                 Tuple[List[Result], Result], self.visit_children(tree)
@@ -1840,7 +2152,7 @@ 

Source code for celpy.evaluation

 
 
 
-[docs] +[docs] @trace def addition(self, tree: lark.Tree) -> Result: """ @@ -1855,7 +2167,7 @@

Source code for celpy.evaluation

 
             values = self.visit_children(tree)
             func = functions[op_name_map[tree.data]]
-            result = func(*values)
+            result_value = func(*values)
 
         The AST doesn't provide a flat list of values, however.
         """
@@ -1866,11 +2178,13 @@ 

Source code for celpy.evaluation

 
         elif len(tree.children) == 2:
             left_op, right_tree = cast(Tuple[lark.Tree, lark.Tree], tree.children)
+            # Map a node data in parse tree to an operation function.
             op_name = {
                 "addition_add": "_+_",
                 "addition_sub": "_-_",
             }[left_op.data]
-            func = self.functions[op_name]
+            # func = self.functions[op_name]    # Refactor ``self.functions`` into an Activation
+            func = self.activation.resolve_function(op_name)
             # NOTE: values have the structure [[left], right]
             (left, *_), right = cast(
                 Tuple[List[Result], Result], self.visit_children(tree)
@@ -1904,7 +2218,7 @@ 

Source code for celpy.evaluation

 
 
 
-[docs] +[docs] @trace def multiplication(self, tree: lark.Tree) -> Result: """ @@ -1920,7 +2234,7 @@

Source code for celpy.evaluation

 
                 values = self.visit_children(tree)
                 func = functions[op_name_map[tree.data]]
-                result = func(*values)
+                result_value = func(*values)
 
         The AST doesn't provide a flat list of values, however.
         """
@@ -1931,12 +2245,14 @@ 

Source code for celpy.evaluation

 
         elif len(tree.children) == 2:
             left_op, right_tree = cast(Tuple[lark.Tree, lark.Tree], tree.children)
+            # Map a node data in parse tree to an operation function.
             op_name = {
                 "multiplication_div": "_/_",
                 "multiplication_mul": "_*_",
                 "multiplication_mod": "_%_",
             }[left_op.data]
-            func = self.functions[op_name]
+            # func = self.functions[op_name]   # Refactor ``self.functions`` into an Activation
+            func = self.activation.resolve_function(op_name)
             # NOTE: values have the structure [[left], right]
             (left, *_), right = cast(
                 Tuple[List[Result], Result], self.visit_children(tree)
@@ -1977,7 +2293,7 @@ 

Source code for celpy.evaluation

 
 
 
-[docs] +[docs] @trace def unary(self, tree: lark.Tree) -> Result: """ @@ -1992,23 +2308,25 @@

Source code for celpy.evaluation

 
             values = self.visit_children(tree)
             func = functions[op_name_map[tree.data]]
-            result = func(*values)
+            result_value = func(*values)
 
         But, values has the structure ``[[], right]``
         """
         if len(tree.children) == 1:
-            # member with no preceeding unary_not or unary_neg
+            # member with no preceding unary_not or unary_neg
             # TODO: If there are two possible values (namespace v. mapping) chose the namespace.
             values = self.visit_children(tree)
             return values[0]
 
         elif len(tree.children) == 2:
             op_tree, right_tree = cast(Tuple[lark.Tree, lark.Tree], tree.children)
+            # Map a node data in parse tree to an operation function.
             op_name = {
                 "unary_not": "!_",
                 "unary_neg": "-_",
             }[op_tree.data]
-            func = self.functions[op_name]
+            # func = self.functions[op_name]    # Refactor ``self.functions`` into an Activation
+            func = self.activation.resolve_function(op_name)
             # NOTE: values has the structure [[], right]
             left, right = cast(Tuple[List[Result], Result], self.visit_children(tree))
             self.logger.debug("unary %s %r", op_name, right)
@@ -2040,7 +2358,7 @@ 

Source code for celpy.evaluation

 
 
 
-[docs] +[docs] def build_macro_eval( self, child: lark.Tree ) -> Callable[[celpy.celtypes.Value], Any]: @@ -2066,25 +2384,25 @@

Source code for celpy.evaluation

         """
         args = cast(lark.Tree, child.children[2])
         var_tree, expr_tree = cast(Tuple[lark.Tree, lark.Tree], args.children)
-        identifier = FindIdent.in_tree(var_tree)
-        if identifier is None:  # pragma: no cover
-            # This seems almost impossible.
-            raise CELSyntaxError(
+        idents = list(var_tree.find_data("ident"))
+        if len(idents) != 1:
+            # Essentially impossible.
+            raise CELSyntaxError(  # pragma: no cover
                 f"{child.data} {child.children}: bad macro node",
                 line=child.meta.line,
                 column=child.meta.column,
             )
-        # nested_eval = Evaluator(ast=expr_tree, activation=self.activation)
+        identifier = cast(lark.Token, idents[0].children[0]).value
         nested_eval = self.sub_evaluator(ast=expr_tree)
 
         def sub_expr(v: celpy.celtypes.Value) -> Any:
-            return nested_eval.set_activation({identifier: v}).evaluate()
+            return nested_eval.evaluate({identifier: v})
 
         return sub_expr
-[docs] +[docs] def build_ss_macro_eval( self, child: lark.Tree ) -> Callable[[celpy.celtypes.Value], Any]: @@ -2101,20 +2419,29 @@

Source code for celpy.evaluation

         """
         args = cast(lark.Tree, child.children[2])
         var_tree, expr_tree = cast(Tuple[lark.Tree, lark.Tree], args.children)
-        identifier = FindIdent.in_tree(var_tree)
-        if identifier is None:  # pragma: no cover
-            # This seems almost impossible.
-            raise CELSyntaxError(
+        idents = list(var_tree.find_data("ident"))
+        if len(idents) != 1:
+            # Essentially impossible.
+            raise CELSyntaxError(  # pragma: no cover
                 f"{child.data} {child.children}: bad macro node",
                 line=child.meta.line,
                 column=child.meta.column,
             )
+        identifier = cast(lark.Token, idents[0].children[0]).value
+        # identifier = FindIdent.in_tree(var_tree)
+        # if identifier is None:  # pragma: no cover
+        #     # This seems almost impossible.
+        #     raise CELSyntaxError(
+        #         f"{child.data} {child.children}: bad macro node",
+        #         line=child.meta.line,
+        #         column=child.meta.column,
+        #     )
         # nested_eval = Evaluator(ast=expr_tree, activation=self.activation)
         nested_eval = self.sub_evaluator(ast=expr_tree)
 
         def sub_expr(v: celpy.celtypes.Value) -> Any:
             try:
-                return nested_eval.set_activation({identifier: v}).evaluate()
+                return nested_eval.evaluate({identifier: v})
             except CELEvalError as ex:
                 return ex
 
@@ -2122,12 +2449,12 @@ 

Source code for celpy.evaluation

 
 
 
-[docs] +[docs] def build_reduce_macro_eval( self, child: lark.Tree ) -> Tuple[Callable[[Result, Result], Result], lark.Tree]: """ - Builds macro function and intiial expression for reduce(). + Builds macro function and initial expression for reduce(). For example @@ -2148,28 +2475,37 @@

Source code for celpy.evaluation

         reduce_var_tree, iter_var_tree, init_expr_tree, expr_tree = cast(
             Tuple[lark.Tree, lark.Tree, lark.Tree, lark.Tree], args.children
         )
-        reduce_ident = FindIdent.in_tree(reduce_var_tree)
-        iter_ident = FindIdent.in_tree(iter_var_tree)
-        if reduce_ident is None or iter_ident is None:  # pragma: no cover
+        reduce_idents = list(reduce_var_tree.find_data("ident"))
+        iter_idents = list(iter_var_tree.find_data("ident"))
+        if len(reduce_idents) != 1 or len(iter_idents) != 1:  # pragma: no cover
             # This seems almost impossible.
             raise CELSyntaxError(
                 f"{child.data} {child.children}: bad macro node",
                 line=child.meta.line,
                 column=child.meta.column,
             )
+        reduce_ident = cast(lark.Token, reduce_idents[0].children[0]).value
+        iter_ident = cast(lark.Token, iter_idents[0].children[0]).value
+        # reduce_ident = FindIdent.in_tree(reduce_var_tree)
+        # iter_ident = FindIdent.in_tree(iter_var_tree)
+        # if reduce_ident is None or iter_ident is None:  # pragma: no cover
+        #     # This seems almost impossible.
+        #     raise CELSyntaxError(
+        #         f"{child.data} {child.children}: bad macro node",
+        #         line=child.meta.line,
+        #         column=child.meta.column,
+        #     )
         # nested_eval = Evaluator(ast=expr_tree, activation=self.activation)
         nested_eval = self.sub_evaluator(ast=expr_tree)
 
         def sub_expr(r: Result, i: Result) -> Result:
-            return nested_eval.set_activation(
-                {reduce_ident: r, iter_ident: i}
-            ).evaluate()
+            return nested_eval.evaluate({reduce_ident: r, iter_ident: i})
 
         return sub_expr, init_expr_tree
-[docs] +[docs] @trace def member(self, tree: lark.Tree) -> Result: """ @@ -2187,7 +2523,7 @@

Source code for celpy.evaluation

 
 
 
-[docs] +[docs] @trace def member_dot(self, tree: lark.Tree) -> Result: """ @@ -2232,50 +2568,50 @@

Source code for celpy.evaluation

 
         - In all other cases, e.f evaluates to an error.
 
-        TODO: implement member "." IDENT for messages.
+        TODO: implement member "." IDENT for protobuf message types.
         """
         member_tree, property_name_token = cast(
             Tuple[lark.Tree, lark.Token], tree.children
         )
         member = self.visit(member_tree)
         property_name = property_name_token.value
-        result: Result
+        result_value: Result
         if isinstance(member, CELEvalError):
-            result = cast(Result, member)
+            result_value = cast(Result, member)
         elif isinstance(member, NameContainer):
             # Navigation through names provided as external run-time bindings.
             # The dict is the value of a Referent that was part of a namespace path.
             if property_name in member:
-                result = cast(Result, member[property_name].value)
+                result_value = cast(Result, member[property_name].value)
             else:
                 err = f"No {property_name!r} in bindings {sorted(member.keys())}"
-                result = CELEvalError(err, KeyError, None, tree=tree)
-        # TODO: Not sure this is needed...
+                result_value = CELEvalError(err, KeyError, None, tree=tree)
         elif isinstance(member, celpy.celtypes.MessageType):
+            # NOTE: Message's don't have a "default None" behavior: they raise an exception.
             self.logger.debug("member_dot(%r, %r)", member, property_name)
-            result = member.get(property_name)
+            result_value = member.get(property_name)
         # TODO: Future Expansion, handle Protobuf message package...
         # elif isinstance(member, celpy.celtypes.PackageType):
         #     if property_name in member:
-        #         result = member[property_name]
+        #         result_value = member[property_name]
         #     else:
         #         err = f"no such message {property_name!r} in package {member}"
-        #         result = CELEvalError(err, KeyError, None, tree=tree)
+        #         result_value = CELEvalError(err, KeyError, None, tree=tree)
         elif isinstance(member, celpy.celtypes.MapType):
-            # Syntactic sugar: a.b is a["b"] when a is a mapping.
+            # Syntactic sugar: a.b is a["b"] when ``a`` is a mapping.
             try:
-                result = member[property_name]
+                result_value = member[property_name]
             except KeyError:
                 err = f"no such member in mapping: {property_name!r}"
-                result = CELEvalError(err, KeyError, None, tree=tree)
+                result_value = CELEvalError(err, KeyError, None, tree=tree)
         else:
             err = f"{member!r} with type: '{type(member)}' does not support field selection"
-            result = CELEvalError(err, TypeError, None, tree=tree)
-        return result
+ result_value = CELEvalError(err, TypeError, None, tree=tree) + return result_value
-[docs] +[docs] @trace def member_dot_arg(self, tree: lark.Tree) -> Result: """ @@ -2297,7 +2633,7 @@

Source code for celpy.evaluation

         - member "." IDENT "(" ")"  -- used for a several timestamp operations.
         """
         sub_expr: CELFunction
-        result: Result
+        result_value: Result
         reduction: Result
         CELBoolFunction = Callable[
             [celpy.celtypes.BoolType, Result], celpy.celtypes.BoolType
@@ -2313,9 +2649,13 @@ 

Source code for celpy.evaluation

             "all",
             "exists",
             "exists_one",
+            # Extensions to CEL...
             "reduce",
             "min",
         }:
+            # TODO: These can be refactored to share the macro_xyz() functions
+            # used by Transpiled code.
+
             member_list = cast(celpy.celtypes.ListType, self.visit(member_tree))
 
             if isinstance(member_list, CELEvalError):
@@ -2326,13 +2666,13 @@ 

Source code for celpy.evaluation

                 mapping = cast(
                     Iterable[celpy.celtypes.Value], map(sub_expr, member_list)
                 )
-                result = celpy.celtypes.ListType(mapping)
-                return result
+                result_value = celpy.celtypes.ListType(mapping)
+                return result_value
 
             elif method_name_token.value == "filter":
                 sub_expr = self.build_macro_eval(tree)
-                result = celpy.celtypes.ListType(filter(sub_expr, member_list))
-                return result
+                result_value = celpy.celtypes.ListType(filter(sub_expr, member_list))
+                return result_value
 
             elif method_name_token.value == "all":
                 sub_expr = self.build_ss_macro_eval(tree)
@@ -2366,6 +2706,7 @@ 

Source code for celpy.evaluation

                 count = sum(1 for value in member_list if bool(sub_expr(value)))
                 return celpy.celtypes.BoolType(count == 1)
 
+            # Not formally part of CEL...
             elif method_name_token.value == "reduce":
                 # Apply a function to reduce the list to a single value.
                 # The `tree` is a `member_dot_arg` construct with (member, method_name, args)
@@ -2375,6 +2716,7 @@ 

Source code for celpy.evaluation

                 reduction = reduce(reduce_expr, member_list, initial_value)
                 return reduction
 
+            # Not formally part of CEL...
             elif method_name_token.value == "min":
                 # Special case of "reduce()"
                 # with <member>.min() -> <member>.reduce(r, i, int_max, r < i ? r : i)
@@ -2396,19 +2738,19 @@ 

Source code for celpy.evaluation

                 member, ident = cast(
                     Tuple[Result, lark.Token], self.visit_children(tree)
                 )
-                result = self.method_eval(member, ident)
+                result_value = self.method_eval(member, ident)
             else:
                 # assert len(tree.children) == 3
                 member, ident, expr_iter = cast(
                     Tuple[Result, lark.Token, Iterable[Result]],
                     self.visit_children(tree),
                 )
-                result = self.method_eval(member, ident, expr_iter)
-            return result
+ result_value = self.method_eval(member, ident, expr_iter) + return result_value
-[docs] +[docs] @trace def member_index(self, tree: lark.Tree) -> Result: """ @@ -2423,7 +2765,8 @@

Source code for celpy.evaluation

 
         Locating an item in a Mapping or List
         """
-        func = self.functions["_[_]"]
+        # func = self.functions["_[_]"]    # Refactor ``self.functions`` into an Activation
+        func = self.activation.resolve_function("_[_]")
         values = self.visit_children(tree)
         member, index = values
         try:
@@ -2450,7 +2793,7 @@ 

Source code for celpy.evaluation

 
 
 
-[docs] +[docs] @trace def member_object(self, tree: lark.Tree) -> Result: """ @@ -2508,7 +2851,7 @@

Source code for celpy.evaluation

 
 
 
-[docs] +[docs] @trace def primary(self, tree: lark.Tree) -> Result: """ @@ -2523,15 +2866,16 @@

Source code for celpy.evaluation

         list_lit       : "[" [exprlist] "]"
         map_lit        : "{" [mapinits] "}"
 
-        TODO: Refactor into separate methods to skip this complex elif chain.
-        top-level :py:meth:`primary` is similar to :py:meth:`method`.
-        Each of the individual rules then works with a tree instead of a child of the
-        primary tree.
+        .. TODO:: Refactor into separate methods to skip these complex elif chain.
+
+            Top-level :py:meth:`primary` is similar to :py:meth:`method`.
+            Each of the individual rules then works with a tree instead of a child of the
+            primary tree.
 
-        This includes function-like macros: has() and dyn().
+        This includes function-like macros: ``has()`` and ``dyn()``.
         These are special cases and cannot be overridden.
         """
-        result: Result
+        result_value: Result
         name_token: lark.Token
         if len(tree.children) != 1:
             raise CELSyntaxError(
@@ -2555,18 +2899,18 @@ 

Source code for celpy.evaluation

             if len(child.children) == 0:
                 # Empty list
                 # TODO: Refactor into type_eval()
-                result = celpy.celtypes.ListType()
+                result_value = celpy.celtypes.ListType()
             else:
                 # exprlist to be packaged as List.
                 values = self.visit_children(child)
-                result = values[0]
-            return result
+                result_value = values[0]
+            return result_value
 
         elif child.data == "map_lit":
             if len(child.children) == 0:
                 # Empty mapping
                 # TODO: Refactor into type_eval()
-                result = celpy.celtypes.MapType()
+                result_value = celpy.celtypes.MapType()
             else:
                 # mapinits (a sequence of key-value tuples) to be packaged as a dict.
                 # OR. An CELEvalError in case of ValueError caused by duplicate keys.
@@ -2574,30 +2918,36 @@ 

Source code for celpy.evaluation

                 # TODO: Refactor into type_eval()
                 try:
                     values = self.visit_children(child)
-                    result = values[0]
+                    result_value = values[0]
                 except ValueError as ex:
-                    result = CELEvalError(ex.args[0], ex.__class__, ex.args, tree=tree)
+                    result_value = CELEvalError(
+                        ex.args[0], ex.__class__, ex.args, tree=tree
+                    )
                 except TypeError as ex:
-                    result = CELEvalError(ex.args[0], ex.__class__, ex.args, tree=tree)
-            return result
+                    result_value = CELEvalError(
+                        ex.args[0], ex.__class__, ex.args, tree=tree
+                    )
+            return result_value
 
         elif child.data in ("dot_ident", "dot_ident_arg"):
             # "." IDENT ["(" [exprlist] ")"]
             # Leading "." means the name is resolved in the root scope **only**.
-            # No searching through alterantive packages.
+            # No searching through alternative packages.
             # len(child) == 1 -- "." IDENT
             # len(child) == 2 -- "." IDENT "(" exprlist ")" -- TODO: Implement dot_ident_arg.
             values = self.visit_children(child)
             name_token = cast(lark.Token, values[0])
             # Should not be a Function, should only be a Result
-            # TODO: implement dot_ident_arg uses function_eval().
+            # TODO: implement dot_ident_arg using ``function_eval()``, which should match this code.
             try:
-                result = cast(
+                result_value = cast(
                     Result, self.ident_value(name_token.value, root_scope=True)
                 )
             except KeyError as ex:
-                result = CELEvalError(ex.args[0], ex.__class__, ex.args, tree=tree)
-            return result
+                result_value = CELEvalError(
+                    ex.args[0], ex.__class__, ex.args, tree=tree
+                )
+            return result_value
 
         elif child.data == "ident_arg":
             # IDENT ["(" [exprlist] ")"]
@@ -2639,14 +2989,14 @@ 

Source code for celpy.evaluation

                 # Should not be a Function.
                 # Generally Result object (i.e., a variable)
                 # Could be an Annotation object (i.e., a type) for protobuf messages
-                result = cast(Result, self.ident_value(name_token.value))
+                result_value = cast(Result, self.ident_value(name_token.value))
             except KeyError as ex:
                 err = (
                     f"undeclared reference to '{name_token}' "
                     f"(in activation '{self.activation}')"
                 )
-                result = CELEvalError(err, ex.__class__, ex.args, tree=tree)
-            return result
+                result_value = CELEvalError(err, ex.__class__, ex.args, tree=tree)
+            return result_value
 
         else:
             raise CELSyntaxError(
@@ -2657,7 +3007,7 @@ 

Source code for celpy.evaluation

 
 
 
-[docs] +[docs] @trace def literal(self, tree: lark.Tree) -> Result: """ @@ -2673,11 +3023,11 @@

Source code for celpy.evaluation

             )
         value_token = cast(lark.Token, tree.children[0])
         try:
-            result: Result
+            result_value: Result
             if value_token.type == "FLOAT_LIT":
-                result = celpy.celtypes.DoubleType(value_token.value)
+                result_value = celpy.celtypes.DoubleType(value_token.value)
             elif value_token.type == "INT_LIT":
-                result = celpy.celtypes.IntType(value_token.value)
+                result_value = celpy.celtypes.IntType(value_token.value)
             elif value_token.type == "UINT_LIT":
                 if not value_token.value[-1].lower() == "u":
                     raise CELSyntaxError(
@@ -2685,15 +3035,17 @@ 

Source code for celpy.evaluation

                         line=tree.meta.line,
                         column=tree.meta.column,
                     )
-                result = celpy.celtypes.UintType(value_token.value[:-1])
+                result_value = celpy.celtypes.UintType(value_token.value[:-1])
             elif value_token.type in ("MLSTRING_LIT", "STRING_LIT"):
-                result = celstr(value_token)
+                result_value = celstr(value_token)
             elif value_token.type == "BYTES_LIT":
-                result = celbytes(value_token)
+                result_value = celbytes(value_token)
             elif value_token.type == "BOOL_LIT":
-                result = celpy.celtypes.BoolType(value_token.value.lower() == "true")
+                result_value = celpy.celtypes.BoolType(
+                    value_token.value.lower() == "true"
+                )
             elif value_token.type == "NULL_LIT":
-                result = None
+                result_value = None
             else:
                 raise CELUnsupportedError(
                     f"{tree.data} {tree.children}: type not implemented",
@@ -2701,13 +3053,13 @@ 

Source code for celpy.evaluation

                     column=value_token.column or tree.meta.column,
                 )
         except ValueError as ex:
-            result = CELEvalError(ex.args[0], ex.__class__, ex.args, tree=tree)
+            result_value = CELEvalError(ex.args[0], ex.__class__, ex.args, tree=tree)
 
-        return result
+ return result_value
-[docs] +[docs] @trace def exprlist(self, tree: lark.Tree) -> Result: """ @@ -2720,12 +3072,12 @@

Source code for celpy.evaluation

         except StopIteration:
             pass
         # There are no CELEvalError values in the result, so we can narrow the domain.
-        result = celpy.celtypes.ListType(cast(List[celpy.celtypes.Value], values))
-        return result
+ result_value = celpy.celtypes.ListType(cast(List[celpy.celtypes.Value], values)) + return result_value
-[docs] +[docs] @trace def fieldinits(self, tree: lark.Tree) -> Result: """ @@ -2752,7 +3104,7 @@

Source code for celpy.evaluation

 
 
 
-[docs] +[docs] @trace def mapinits(self, tree: lark.Tree) -> Result: """ @@ -2761,22 +3113,1129 @@

Source code for celpy.evaluation

         Extract the key expr's and value expr's to a list of pairs.
         This raises an exception on a duplicate key.
 
-        TODO: Is ``{'a': 1, 'b': 2/0}['a']`` a meaningful result in CEL?
-        Or is this an error because the entire member is erroneous?
+        .. TODO:: CEL question.
 
+            Is ``{'a': 1, 'b': 2/0}['a']`` a meaningful result in CEL?
+            Or is this an error because the entire member object is erroneous?
+
+        ..  TODO:: Refactor to use MapType([(key, value),...]) init, which checks for duplicates.
+
+            This simplifies to ``celpy.celtypes.MapType(pairs)``
         """
-        result = celpy.celtypes.MapType()
+        result_value = celpy.celtypes.MapType()
 
         # Not sure if this cast is sensible. Should a CELEvalError propagate up from the
         # sub-expressions? See the error check in :py:func:`exprlist`.
         keys_values = cast(List[celpy.celtypes.Value], self.visit_children(tree))
         pairs = zip(keys_values[0::2], keys_values[1::2])
         for key, value in pairs:
-            if key in result:
+            if key in result_value:
                 raise ValueError(f"Duplicate key {key!r}")
-            result[key] = value
+            result_value[key] = value
+
+        return result_value
+
+ + + +# GLOBAL activation used by Transpiled code. +# This slightly simplifies the exception handling, by using a 1-argument function +# to compute a Value or a CELEvalError. + +the_activation: Activation + + +
+[docs] +def result(activation: Activation, cel_expr: Callable[[Activation], Result]) -> Result: + """ + Implements "checked exception" handling for CEL expressions transpiled to Python. + An expression must be wrapped by a lambda. + The lambda is evaluated by this function; a subset of Python exceptions become ``CELEvalError`` objects. + + >>> some_activation = Activation() + + Within the CEL transpiled code, we can now use code like this... + + >>> expr = lambda activation: 355 / 0 + >>> result(some_activation, expr) + CELEvalError(*('divide by zero', <class 'ZeroDivisionError'>, ('division by zero',))) + + The exception becomes an object. + """ + + value: Result + try: + value = cel_expr(activation) + except ( + ValueError, + KeyError, + TypeError, + ZeroDivisionError, + OverflowError, + IndexError, + NameError, + ) as ex: + ex_message = { + ValueError: "return error for overflow", + KeyError: f"no such member in mapping: {ex.args[0]!r}", + TypeError: "no such overload", + ZeroDivisionError: "divide by zero", + OverflowError: "return error for overflow", + IndexError: "invalid_argument", + UnicodeDecodeError: "invalid UTF-8", + NameError: f"undeclared reference to {ex.args[0]!r} (in container {ex.args[1:]!r})", + }[ex.__class__] + _, _, tb = sys.exc_info() + value = CELEvalError(ex_message, ex.__class__, ex.args).with_traceback(tb) + value.__cause__ = ex + logger.debug("result = %r", value) + return value
+ + + +
+[docs] +def macro_map( + activation: Activation, + bind_variable: str, + cel_expr: Callable[[Activation], celpy.celtypes.Value], + cel_gen: Callable[[Activation], Iterable[Activation]], +) -> Result: + """The results of a source.map(v, expr) macro: a list of values.""" + activations = ( + activation.nested_activation(vars={bind_variable: cast(Result, _value)}) + for _value in cel_gen(activation) + ) + return celpy.celtypes.ListType(map(cel_expr, activations))
+ + + +
+[docs] +def macro_filter( + activation: Activation, + bind_variable: str, + cel_expr: Callable[[Activation], celpy.celtypes.Value], + cel_gen: Callable[[Activation], Iterable[Activation]], +) -> Result: + """The results of a source.filter(v, expr) macro: a list of values.""" + r: list[celpy.celtypes.Value] = [] + for value in cel_gen(activation): + f = cel_expr( + activation.nested_activation(vars={bind_variable: cast(Result, value)}) + ) + if bool(f): + r.append(cast(celpy.celtypes.Value, value)) + return celpy.celtypes.ListType(iter(r))
+ + + +
+[docs] +def macro_exists_one( + activation: Activation, + bind_variable: str, + cel_expr: Callable[[Activation], celpy.celtypes.Value], + cel_gen: Callable[[Activation], Iterable[Activation]], +) -> Result: + """The results of a source.exists_one(v, expr) macro: a list of values. + + Note the short-circuit concept. + Count the True; Break on an Exception + """ + count = 0 + activations = ( + activation.nested_activation(vars={bind_variable: cast(Result, _value)}) + for _value in cel_gen(activation) + ) + for result in filter(cel_expr, activations): + count += 1 if bool(result) else 0 + return celpy.celtypes.BoolType(count == 1)
+ + + +
+[docs] +def macro_exists( + activation: Activation, + bind_variable: str, + cel_expr: Callable[[Activation], celpy.celtypes.Value], + cel_gen: Callable[[Activation], Iterable[Activation]], +) -> Result: + """The results of a source.exists(v, expr) macro: a list of values.""" + activations = ( + activation.nested_activation(vars={bind_variable: cast(Result, _value)}) + for _value in cel_gen(activation) + ) + return celpy.celtypes.BoolType( + reduce( + cast( + Callable[[celpy.celtypes.BoolType, Result], celpy.celtypes.BoolType], + celpy.celtypes.logical_or, + ), + (result(act, cel_expr) for act in activations), + celpy.celtypes.BoolType(False), + ) + )
+ + + +
+[docs] +def macro_all( + activation: Activation, + bind_variable: str, + cel_expr: Callable[[Activation], celpy.celtypes.Value], + cel_gen: Callable[[Activation], Iterable[Activation]], +) -> Result: + """The results of a source.all(v, expr) macro: a list of values.""" + activations = ( + activation.nested_activation(vars={bind_variable: cast(Result, _value)}) + for _value in cel_gen(activation) + ) + return celpy.celtypes.BoolType( + reduce( + cast( + Callable[[celpy.celtypes.BoolType, Result], celpy.celtypes.BoolType], + celpy.celtypes.logical_and, + ), + (result(act, cel_expr) for act in activations), + celpy.celtypes.BoolType(True), + ) + )
+ + + +
+[docs] +class TranspilerTree(lark.Tree): + data: str + children: "Sequence[Union[lark.Token, TranspilerTree]]" # type: ignore[assignment] + +
+[docs] + def __init__( + self, + data: str, + children: "Sequence[Union[lark.Token, TranspilerTree]]", + meta: Optional[lark.tree.Meta] = None, + ) -> None: + super().__init__(data, children, meta) # type: ignore [arg-type] + self.expr_number: int = 0 # Updated by visitor + self.transpiled: str = ( + f"ex_{self.expr_number}(activation)" # Default, often replaced. + ) + self.checked_exception: Union[ + tuple[Template, dict[str, Callable[[TranspilerTree], str]]], None + ] = None # Optional
+
+ + + +
+[docs] +class Transpiler: + """ + Transpile the CEL construct(s) to Python functions. + This is a **Facade** that wraps two visitor subclasses to do two phases + of transpilation. + + The resulting Python code can be used with ``compile()`` and ``exec()``. + + :Phase I: + The easy transpilation. + It builds simple text expressions for each node of the AST. + This sets aside exception-checking code including short-circuit logic operators and macros. + This decorates the AST with transpiled Python where possible. + It can also decorate with ``Template`` objects that require text from children. + + :Phase II: + Collects a sequence of statements. + All of the exception-checking for short-circuit logic operators and macros is packaged as lambdas + that may (or may not) be evaluated. + + Ideally, there could be a ``Transpiler`` ABC, and the ``PythonTranspiler`` defined as a subclass. + Pragmatically, we can't see any other sensible transpilation. + + .. rubric:: Exception Checking + + To handle ``2 / 0 || true``, the ``||``, ``&&``, and ``?:`` operators + the generated code creates lambdas to avoid execution where possible. + An alternative is a Monad-like structure to bottle up an exception, silencing it if it's unused. + + .. rubric:: Identifiers + + Identifiers have three meanings: + + - An object. This is either a variable provided in an ``Activation`` or a function provided + when building an execution. Objects also have type annotations. + + - A type annotation without an object. This can used to build protobuf messages. + + - A macro name. The ``member_dot_arg`` construct (e.g., ``member.map(v, expr)``) may have a macro instead of a method. + Plus the ``ident_arg`` construct may be a ``dyn()`` or ``has()`` macro instead of a function + + Other than macros, a name maps to an ``Referent`` instance. + This will have an annotation and -- perhaps -- an associated object. + + .. important MACROS ARE SPECIAL + + They aren't simple functions. + + The macros do not simply visit their children to perform evaluation. + + There's a bind variable and a function with the bind variable. + This isn't **trivially** moved from expression stack to statements. + + There are two functions that are macro-like: + + - ``dyn()`` does effectively nothing. + It visits its children, but also provides progressive type resolution + through detailed type annotation of the AST. + + - ``has()`` attempts to visit the child and does a boolean transformation + on the resulting exception or value. + This is a macro because it doesn't raise the exception for a missing + member item reference, but instead maps any exception to ``False``. + It doesn't return the value found for a member item reference; instead, it maps + successfully finding a member to ``True``. + + The member and expression list of a macro are transformed into lambdas for use by + special ``macro_{name}`` functions. These functions provided the necessary generator + expression to provide CEL semantics. + + Names have nested paths. For example, ``a.b.c`` is a mapping ``a``, that contains a mapping, ``b``, + that contains a name ``c``. + + The :py:meth:`member` method implements the macro evaluation behavior. + It does not **always** trivially descend into the children. + In the case of macros, the member evaluates one child tree in the presence + of values from another child tree using specific variable binding in a kind + of stack frame. + """ + + logger = logging.getLogger("celpy.Transpiler") + +
+[docs] + def __init__( + self, + ast: TranspilerTree, + activation: Activation, + ) -> None: + """ + Create the Transpiler for an AST with specific variables and functions. + + :param ast: The AST to transpile. + :param activation: An activation with functions and types to use. + """ + self.ast = ast + self.base_activation = activation + self.activation = self.base_activation + + self.logger.debug("Transpiler activation: %r", self.activation)
- return result
+ # self.logger.debug("functions: %r", self.functions) # Refactor ``self.functions`` into an Activation + +
+[docs] + def transpile(self) -> None: + """Two-phase transpilation. + + 1. Decorate AST with the most constructs. + 2. Expand into statements for lambdas that wrap checked exceptions. + """ + phase_1 = Phase1Transpiler(self) + phase_1.visit(self.ast) + + phase_2 = Phase2Transpiler(self) + phase_2.visit(self.ast) + + statements = phase_2.statements(self.ast) + + # The complete sequence of statements and the code object. + self.source_text = "\n".join(statements) + self.executable_code = compile(self.source_text, "<string>", "exec")
+ + +
+[docs] + def evaluate(self, context: Context) -> celpy.celtypes.Value: + if context: + self.activation = self.base_activation.clone() + self.activation.identifiers.load_values(context) + else: + self.activation = self.base_activation + self.logger.debug("Activation: %r", self.activation) + + # Global for the top-level ``CEL = result(base_activation, ...)`` statement. + evaluation_globals = ( + celpy.evaluation.result.__globals__ + ) # the ``evaluation`` moodule + evaluation_globals["base_activation"] = self.activation + try: + exec(self.executable_code, evaluation_globals) + value = cast(celpy.celtypes.Value, evaluation_globals["CEL"]) + if isinstance(value, CELEvalError): + raise value + return value + except Exception as ex: + # A Python problem during ``exec()`` + self.logger.error("Internal error: %r", ex) + raise CELEvalError("evaluation error", type(ex), ex.args)
+
+ + + +
+[docs] +class Phase1Transpiler(lark.visitors.Visitor_Recursive): + """ + Decorate all nodes with transpiled Python code, where possible. + For short-circuit operators or macros, where a "checked exception" is required, + a simple ``ex_{n}`` name is present, and separate statements are provided as + a decoration to handle the more complicated cases. + + Each construct has an associated ``Template``. + For the simple cases, the transpiled value is the entire expression. + + >>> from unittest.mock import Mock + >>> source = "7 * (3 + 3)" + >>> parser = celpy.CELParser() + >>> tree = parser.parse(source) + >>> tp = Phase1Transpiler(Mock(base_activation=celpy.Activation())) + >>> _ = tp.visit(tree) + >>> tree.transpiled + 'operator.mul(celpy.celtypes.IntType(7), operator.add(celpy.celtypes.IntType(3), celpy.celtypes.IntType(3)))' + + Some constructs wrap macros or short-circuit logic, and require a more sophisticated execution. + There will be "checked exceptions", returned as values. + This requires statements with lambdas that can be wrapped by the ``result()`` function. + + The ``Phase2Transpiler`` does this transformation from expressions to a sequence of statements. + """ + +
+[docs] + def __init__(self, facade: Transpiler) -> None: + self.facade = facade + self.activation = facade.base_activation + self.expr_number = 0
+ + +
+[docs] + def visit(self, tree: TranspilerTree) -> TranspilerTree: # type: ignore[override] + """Initialize the decorations for each node.""" + tree.expr_number = self.expr_number + # tree.transpiled = f"ex_{tree.expr_number}(activation)" # Default, will be replaced. + # tree.checked_exception: Union[str, None] = None # Optional + self.expr_number += 1 + return super().visit(tree) # type: ignore[return-value]
+ + +
+[docs] + def func_name(self, label: str) -> str: + """ + Provide a transpiler-friendly name for the function. + + Some internally-defined functions appear to come from ``_operator`` module. + We need to rename some ``celpy`` functions to be from ``operator``. + + Some functions -- specifically lt, le, gt, ge, eq, ne -- are wrapped ``boolean(operator.f)`` + obscuring their name. + """ + try: + # func = self.functions[label] # Refactor ``self.functions`` into an Activation + func = self.activation.resolve_function(label) + except KeyError: + return f"CELEvalError('unbound function', KeyError, ({label!r},))" + module = {"_operator": "operator"}.get(func.__module__, func.__module__) + return f"{module}.{func.__qualname__}"
+ + +
+[docs] + def expr(self, tree: TranspilerTree) -> None: + """ + expr : conditionalor ["?" conditionalor ":" expr] + """ + if len(tree.children) == 1: + tree.transpiled = cast(TranspilerTree, tree.children[0]).transpiled + elif len(tree.children) == 3: + template = Template( + dedent("""\ + # expr: + ex_${n}_c = lambda activation: ${cond} + ex_${n}_l = lambda activation: ${left} + ex_${n}_r = lambda activation: ${rght} + ex_${n} = lambda activation: ${func_name}(celpy.evaluation.result(activation, ex_${n}_c), celpy.evaluation.result(activation, ex_${n}_l), celpy.evaluation.result(activation, ex_${n}_r))""") + ) + tree.checked_exception = ( + template, + dict( + n=lambda tree: str(tree.expr_number), + func_name=lambda tree: self.func_name("_?_:_"), + cond=lambda tree: cast(TranspilerTree, tree.children[0]).transpiled, + left=lambda tree: cast(TranspilerTree, tree.children[1]).transpiled, + rght=lambda tree: cast(TranspilerTree, tree.children[2]).transpiled, + ), + ) + tree.transpiled = f"ex_{tree.expr_number}(activation)"
+ + +
+[docs] + def conditionalor(self, tree: TranspilerTree) -> None: + """ + conditionalor : [conditionalor "||"] conditionaland + """ + if len(tree.children) == 1: + tree.transpiled = cast(TranspilerTree, tree.children[0]).transpiled + elif len(tree.children) == 2: + template = Template( + dedent("""\ + # conditionalor: + ex_${n}_l = lambda activation: ${left} + ex_${n}_r = lambda activation: ${rght} + ex_${n} = lambda activation: ${func_name}(celpy.evaluation.result(activation, ex_${n}_l), celpy.evaluation.result(activation, ex_${n}_r))""") + ) + tree.checked_exception = ( + template, + dict( + n=lambda tree: str(tree.expr_number), + func_name=lambda tree: self.func_name("_||_"), + left=lambda tree: cast(TranspilerTree, tree.children[0]).transpiled, + rght=lambda tree: cast(TranspilerTree, tree.children[1]).transpiled, + ), + ) + tree.transpiled = f"ex_{tree.expr_number}(activation)"
+ + +
+[docs] + def conditionaland(self, tree: TranspilerTree) -> None: + """ + conditionaland : [conditionaland "&&"] relation + """ + if len(tree.children) == 1: + tree.transpiled = cast(TranspilerTree, tree.children[0]).transpiled + elif len(tree.children) == 2: + template = Template( + dedent("""\ + # conditionaland: + ex_${n}_l = lambda activation: ${left} + ex_${n}_r = lambda activation: ${rght} + ex_${n} = lambda activation: ${func_name}(celpy.evaluation.result(activation, ex_${n}_l), celpy.evaluation.result(activation, ex_${n}_r))""") + ) + tree.checked_exception = ( + template, + dict( + n=lambda tree: str(tree.expr_number), + func_name=lambda tree: self.func_name("_&&_"), + left=lambda tree: cast(TranspilerTree, tree.children[0]).transpiled, + rght=lambda tree: cast(TranspilerTree, tree.children[1]).transpiled, + ), + ) + tree.transpiled = f"ex_{tree.expr_number}(activation)"
+ + +
+[docs] + def relation(self, tree: TranspilerTree) -> None: + """ + relation : [relation_lt | relation_le | relation_ge | relation_gt + | relation_eq | relation_ne | relation_in] addition + + relation_lt : relation "<" + relation_le : relation "<=" + relation_gt : relation ">" + relation_ge : relation ">=" + relation_eq : relation "==" + relation_ne : relation "!=" + relation_in : relation "in" + """ + if len(tree.children) == 1: + tree.transpiled = cast(TranspilerTree, tree.children[0]).transpiled + elif len(tree.children) == 2: + left_op, right_tree = cast( + Tuple[TranspilerTree, TranspilerTree], tree.children + ) + op_name = { + "relation_lt": "_<_", + "relation_le": "_<=_", + "relation_ge": "_>=_", + "relation_gt": "_>_", + "relation_eq": "_==_", + "relation_ne": "_!=_", + "relation_in": "_in_", + }[left_op.data] + func_name = self.func_name(op_name) + template = Template("${func_name}(${left}, ${right})") + tree.transpiled = template.substitute( + func_name=func_name, + left=cast(TranspilerTree, left_op.children[0]).transpiled, + right=right_tree.transpiled, + )
+ + +
+[docs] + def addition(self, tree: TranspilerTree) -> None: + """ + addition : [addition_add | addition_sub] multiplication + + addition_add : addition "+" + addition_sub : addition "-" + """ + if len(tree.children) == 1: + tree.transpiled = cast(TranspilerTree, tree.children[0]).transpiled + elif len(tree.children) == 2: + left_op, right_tree = cast( + Tuple[TranspilerTree, TranspilerTree], tree.children + ) + op_name = { + "addition_add": "_+_", + "addition_sub": "_-_", + }[left_op.data] + func_name = self.func_name(op_name) + template = Template("${func_name}(${left}, ${right})") + tree.transpiled = template.substitute( + func_name=func_name, + left=cast(TranspilerTree, left_op.children[0]).transpiled, + right=right_tree.transpiled, + )
+ + +
+[docs] + def multiplication(self, tree: TranspilerTree) -> None: + """ + multiplication : [multiplication_mul | multiplication_div | multiplication_mod] unary + + multiplication_mul : multiplication "*" + multiplication_div : multiplication "/" + multiplication_mod : multiplication "%" + """ + if len(tree.children) == 1: + tree.transpiled = cast(TranspilerTree, tree.children[0]).transpiled + elif len(tree.children) == 2: + template = Template("${func_name}(*${children})") + left_op, right_tree = cast( + Tuple[TranspilerTree, TranspilerTree], tree.children + ) + op_name = { + "multiplication_mul": "_*_", + "multiplication_div": "_/_", + "multiplication_mod": "_%_", + }[left_op.data] + func_name = self.func_name(op_name) + template = Template("${func_name}(${left}, ${right})") + tree.transpiled = template.substitute( + func_name=func_name, + left=cast(TranspilerTree, left_op.children[0]).transpiled, + right=right_tree.transpiled, + )
+ + +
+[docs] + def unary(self, tree: TranspilerTree) -> None: + """ + unary : [unary_not | unary_neg] member + + unary_not : "!" + unary_neg : "-" + """ + if len(tree.children) == 1: + tree.transpiled = cast(TranspilerTree, tree.children[0]).transpiled + elif len(tree.children) == 2: + template = Template("${func_name}(${children})") + op_tree, right_tree = cast( + Tuple[TranspilerTree, TranspilerTree], tree.children + ) + op_name = { + "unary_not": "!_", + "unary_neg": "-_", + }[op_tree.data] + func_name = self.func_name(op_name) + children = right_tree.transpiled + tree.transpiled = template.substitute( + func_name=func_name, children=children + )
+ + +
+[docs] + def member(self, tree: TranspilerTree) -> None: + """ + member : member_dot | member_dot_arg | member_item | member_object | primary + """ + tree.transpiled = cast(TranspilerTree, tree.children[0]).transpiled
+ + +
+[docs] + def member_dot(self, tree: TranspilerTree) -> None: + """ + member_dot : member "." IDENT + + .. important:: + + The ``member`` can be any of a variety of objects: + + - ``NameContainer(Dict[str, Referent])`` + + - ``Activation`` + + - ``MapType(Dict[Value, Value])`` + + - ``MessageType(MapType)`` + + All of which define a ``get()`` method. + The nuance is the ``NameContainer`` is also a Python ``dict`` and there's an + overload issue between that ``get()`` and other ``get()`` definitions. + + .. todo:: Define a new get_name(member, 'name') function do this, avoiding the ``get()`` method. + """ + member_tree, property_name_token = cast( + Tuple[TranspilerTree, lark.Token], tree.children + ) + template = Template("${left}.get('${right}')") + tree.transpiled = template.substitute( + left=member_tree.transpiled, right=property_name_token.value + )
+ + +
+[docs] + def member_dot_arg(self, tree: TranspilerTree) -> None: + """ + member_dot_arg : member "." IDENT "(" [exprlist] ")" + + Two flavors: macro and non-macro. + """ + exprlist: Union[TranspilerTree, None] + if len(tree.children) == 3: + member_tree, property_name_token, exprlist = cast( + Tuple[TranspilerTree, lark.Token, TranspilerTree], tree.children + ) + else: + # len(tree.children) == 2, no [exprlist]. + member_tree, property_name_token = cast( + Tuple[TranspilerTree, lark.Token], tree.children + ) + exprlist = None + if property_name_token.value in { + "map", + "filter", + "all", + "exists", + "exists_one", + "reduce", + "min", + }: + # Macro. Defer to Phase II. + template = Template( + dedent("""\ + # member_dot_arg ${macro}: + ex_${n}_l = lambda activation: ${member} + ex_${n}_x = lambda activation: ${expr} + ex_${n} = lambda activation: celpy.evaluation.macro_${macro}(activation, '${bind_variable}', ex_${n}_x, ex_${n}_l) + """) + ) + if len(tree.children) == 3: + # Hackery. Undo the transpiling of the identifier and extract only the name. + context, bind_variable = cast( + TranspilerTree, cast(TranspilerTree, tree.children[2]).children[0] + ).transpiled.split(".") + else: + raise CELSyntaxError( # pragma: no cover + "no bind variable in {property_name_token.value} macro", + line=tree.meta.line, + column=tree.meta.column, + ) + + tree.checked_exception = ( + template, + dict( + n=lambda tree: str(tree.expr_number), + macro=lambda tree: property_name_token.value, + member=lambda tree: cast( + TranspilerTree, tree.children[0] + ).transpiled, + bind_variable=lambda tree: bind_variable, + expr=lambda tree: cast( + TranspilerTree, + cast(TranspilerTree, tree.children[2]).children[1], + ).transpiled + if len(tree.children) == 3 + else "", + ), + ) + tree.transpiled = f"ex_{tree.expr_number}(activation)" + + else: + # Non-macro method name. + if exprlist: + template = Template("${func_name}(${left}, ${right})") + func_name = self.func_name(property_name_token.value) + tree.transpiled = template.substitute( + func_name=func_name, + left=member_tree.transpiled, + right=exprlist.transpiled, + ) + else: + template = Template("${func_name}(${left})") + func_name = self.func_name(property_name_token.value) + tree.transpiled = template.substitute( + func_name=func_name, + left=member_tree.transpiled, + )
+ + +
+[docs] + def member_index(self, tree: TranspilerTree) -> None: + """ + member_item : member "[" expr "]" + """ + template = Template("${func_name}(${member}, ${expr})") + member, expr = cast(tuple[TranspilerTree, TranspilerTree], tree.children) + func_name = self.func_name("_[_]") + tree.transpiled = template.substitute( + func_name=func_name, + member=member.transpiled, + expr=expr.transpiled, + )
+ + +
+[docs] + def member_object(self, tree: TranspilerTree) -> None: + """ + member_object : member "{" [fieldinits] "}" + """ + template = Template("${type_name}([${fieldinits}])") + member = cast(TranspilerTree, tree.children[0]) + fieldinits: str + if len(tree.children) == 2: + fieldinits = cast(TranspilerTree, tree.children[1]).transpiled + else: + fieldinits = "" + type_name = member.transpiled + tree.transpiled = template.substitute( + type_name=type_name, fieldinits=fieldinits + )
+ + +
+[docs] + def primary(self, tree: TranspilerTree) -> None: + """ + primary : dot_ident_arg | dot_ident | ident_arg | ident + | paren_expr | list_lit | map_lit | literal + """ + tree.transpiled = cast(TranspilerTree, tree.children[0]).transpiled
+ + +
+[docs] + def dot_ident_arg(self, tree: TranspilerTree) -> None: + """ + dot_ident_arg : "." IDENT "(" [exprlist] ")" + """ + template = Template("activation.resolve_variable('${ident}')(${exprlist})") + ident = cast(lark.Token, tree.children[0]).value + if len(tree.children) == 2: + exprlist = cast(TranspilerTree, tree.children[1]).transpiled + else: + exprlist = "" + tree.transpiled = template.substitute(ident=ident, exprlist=exprlist)
+ + +
+[docs] + def dot_ident(self, tree: TranspilerTree) -> None: + """ + dot_ident : "." IDENT + """ + template = Template("activation.resolve_variable('${ident}')") + ident = cast(lark.Token, tree.children[0]).value + tree.transpiled = template.substitute(ident=ident)
+ + +
+[docs] + def ident_arg(self, tree: TranspilerTree) -> None: + """ + ident_arg : IDENT "(" [exprlist] ")" + """ + op = cast(lark.Token, tree.children[0]).value + if len(tree.children) == 2: + exprlist = cast(TranspilerTree, tree.children[1]).transpiled + else: + exprlist = "" + if op in {"has", "dyn"}: + # Macro-like has() or dyn() + if op == "dyn": + tree.transpiled = cast(TranspilerTree, tree.children[1]).transpiled + elif op == "has": + # try to evaluate the exprlist expression + # TODO: as macro_has() would be better... + template = Template( + dedent("""\ + # ident_arg has: + ex_${n}_h = lambda activation: ${exprlist} + ex_${n} = lambda activation: not isinstance(celpy.evaluation.result(activation, ex_${n}_h), CELEvalError) + """) + ) + tree.checked_exception = ( + template, + dict( + n=lambda tree: str(tree.expr_number), + exprlist=lambda tree: cast( + TranspilerTree, tree.children[1] + ).transpiled, + ), + ) + tree.transpiled = f"ex_{tree.expr_number}(activation)" + else: + # Other function + template = Template("${func_name}(${exprlist})") + func_name = self.func_name(op) + if len(tree.children) == 2: + exprlist = cast(TranspilerTree, tree.children[1]).transpiled + else: + exprlist = "" + tree.transpiled = template.substitute( + func_name=func_name, exprlist=exprlist + )
+ + +
+[docs] + def ident(self, tree: TranspilerTree) -> None: + """ + ident : IDENT + """ + template = Template("activation.${ident}") + tree.transpiled = template.substitute( + ident=cast(lark.Token, tree.children[0]).value + )
+ + +
+[docs] + def paren_expr(self, tree: TranspilerTree) -> None: + """ + paren_expr : "(" expr ")" + """ + tree.transpiled = cast(Sequence[TranspilerTree], tree.children)[0].transpiled
+ + +
+[docs] + def list_lit(self, tree: TranspilerTree) -> None: + """ + list_lit : "[" [exprlist] "]" + """ + if tree.children: + exprlist = cast(Sequence[TranspilerTree], tree.children)[0].transpiled + else: + exprlist = "" + template = Template("celpy.celtypes.ListType([${exprlist}])") + tree.transpiled = template.substitute(exprlist=exprlist)
+ + +
+[docs] + def map_lit(self, tree: TranspilerTree) -> None: + """ + map_lit : "{" [mapinits] "}" + """ + if tree.children: + mapinits = cast(Sequence[TranspilerTree], tree.children)[0].transpiled + else: + mapinits = "" + template = Template("celpy.celtypes.MapType([${mapinits}])") + tree.transpiled = template.substitute(mapinits=mapinits)
+ + +
+[docs] + def exprlist(self, tree: TranspilerTree) -> None: + """ + exprlist : expr ("," expr)* + """ + exprs = ", ".join( + c.transpiled for c in cast(Sequence[TranspilerTree], tree.children) + ) + tree.transpiled = exprs
+ + +
+[docs] + def fieldinits(self, tree: TranspilerTree) -> None: + """ + fieldinits : IDENT ":" expr ("," IDENT ":" expr)* + """ + idents = [ + cast(lark.Token, c).value + for c in cast(Sequence[TranspilerTree], tree.children)[::2] + ] + exprs = [ + c.transpiled for c in cast(Sequence[TranspilerTree], tree.children)[1::2] + ] + assert len(idents) == len(exprs), "Invalid AST" + items = ", ".join(f"('{n}', {v})" for n, v in zip(idents, exprs)) + tree.transpiled = items
+ + +
+[docs] + def mapinits(self, tree: TranspilerTree) -> None: + """ + mapinits : expr ":" expr ("," expr ":" expr)* + """ + keys = [ + c.transpiled for c in cast(Sequence[TranspilerTree], tree.children)[::2] + ] + values = [ + c.transpiled for c in cast(Sequence[TranspilerTree], tree.children)[1::2] + ] + assert len(keys) == len(values) + items = ", ".join(f"({k}, {v})" for k, v in zip(keys, values)) + tree.transpiled = items
+ + +
+[docs] + def literal(self, tree: TranspilerTree) -> None: + """ + literal : UINT_LIT | FLOAT_LIT | INT_LIT | MLSTRING_LIT | STRING_LIT | BYTES_LIT + | BOOL_LIT | NULL_LIT + """ + value_token = cast(lark.Token, tree.children[0]) + if value_token.type == "FLOAT_LIT": + lit_text = f"celpy.celtypes.DoubleType({value_token.value})" + elif value_token.type == "INT_LIT": + lit_text = f"celpy.celtypes.IntType({value_token.value})" + elif value_token.type == "UINT_LIT": + if not value_token.value[-1].lower() == "u": + raise CELSyntaxError( + f"invalid unsigned int literal {value_token!r}", + line=tree.meta.line, + column=tree.meta.column, + ) # pragma: no cover + lit_text = f"celpy.celtypes.UintType({value_token.value[:-1]})" + elif value_token.type in ("MLSTRING_LIT", "STRING_LIT"): + lit_text = f"celpy.celtypes.{celstr(value_token)!r}" + elif value_token.type == "BYTES_LIT": + lit_text = f"celpy.celtypes.{celbytes(value_token)!r}" + elif value_token.type == "BOOL_LIT": + lit_text = f"celpy.celtypes.BoolType({value_token.value.lower() == 'true'})" + elif value_token.type == "NULL_LIT": + lit_text = "None" # Not celpy.celtypes.NullType() in transpiled code. + else: + raise CELUnsupportedError( + f"{tree.data} {tree.children}: type not implemented", + line=value_token.line or tree.meta.line, + column=value_token.column or tree.meta.column, + ) # pragma: no cover + tree.transpiled = lit_text
+
+ + + +
+[docs] +class Phase2Transpiler(lark.visitors.Visitor_Recursive): + """ + Extract any checked_exception evaluation statements that decorate the parse tree. + Also, get the overall top-level expression, assigned to special variable, CEL. + + >>> from unittest.mock import Mock + >>> from pprint import pprint + >>> source = '["hello", "world"].map(x, x) == ["hello", "world"]' + >>> celpy.CELParser.CEL_PARSER = None + >>> parser = celpy.CELParser(tree_class=celpy.evaluation.TranspilerTree) + >>> tree = parser.parse(source) + >>> tp1 = Phase1Transpiler(Mock(base_activation=celpy.Activation())) + >>> _ = tp1.visit(tree) + >>> tp2 = Phase2Transpiler(Mock(base_activation=celpy.Activation())) + >>> _ = tp2.visit(tree) + >>> pprint(tp2.statements(tree), width=256) + ['# member_dot_arg map:', + "ex_10_l = lambda activation: celpy.celtypes.ListType([celpy.celtypes.StringType('hello'), celpy.celtypes.StringType('world')])", + 'ex_10_x = lambda activation: activation.x', + "ex_10 = lambda activation: celpy.evaluation.macro_map(activation, 'x', ex_10_x, ex_10_l)", + "CEL = celpy.evaluation.result(base_activation, lambda activation: celpy.evaluation.bool_eq(ex_10(activation), celpy.celtypes.ListType([celpy.celtypes.StringType('hello'), celpy.celtypes.StringType('world')])))\\n"] + + """ + +
+[docs] + def __init__(self, facade: Transpiler) -> None: + self.facade = facade + self._statements: list[str] = []
+ + +
+[docs] + def expr(self, tree: TranspilerTree) -> None: + """ + expr : conditionalor ["?" conditionalor ":" expr] + + All checked_exception structures are a tuple[Template, dict[str, Callable[[Tree], str]] + """ + if tree.checked_exception: + template, bindings = tree.checked_exception + self._statements.extend( + template.substitute( + {k: v(tree) for k, v in bindings.items()} + ).splitlines() + )
+ + + conditionalor = expr + conditionaland = expr + member_dot_arg = expr + ident_arg = expr + +
+[docs] + def statements(self, tree: TranspilerTree) -> list[str]: + """ + Appends the final CEL = ... statement to the sequence of statements, + and returns the transpiled code. + + Two patterns: + + 1. Top-most expr was a deferred template, and already is a lambda. + It will be a string of the form ``"ex_\\d+(activation)"``. + + 2. Top-most expr was **not** a deferred template, and needs a lambda wrapper. + It will **not** be a simple ``"ex_\\d+"`` reference. + """ + expr_pattern = re.compile(r"^(ex_\d+)\(\w+\)$") + if match := expr_pattern.match(tree.transpiled): + template = Template( + dedent("""\ + CEL = celpy.evaluation.result(base_activation, ${lambda_name}) + """) + ) + final = template.substitute(n=tree.expr_number, lambda_name=match.group(1)) + else: + template = Template( + dedent("""\ + CEL = celpy.evaluation.result(base_activation, lambda activation: ${expr}) + """) + ) + final = template.substitute(n=tree.expr_number, expr=tree.transpiled) + return self._statements + [final]
@@ -2801,7 +4260,7 @@

Source code for celpy.evaluation

 
 
 
-[docs] +[docs] def celstr(token: lark.Token) -> celpy.celtypes.StringType: """ Evaluate a CEL string literal, expanding escapes to create a Python string. @@ -2852,7 +4311,7 @@

Source code for celpy.evaluation

 
 
 
-[docs] +[docs] def celbytes(token: lark.Token) -> celpy.celtypes.BytesType: """ Evaluate a CEL bytes literal, expanding escapes to create a Python bytes object. @@ -2925,14 +4384,15 @@

CEL in Python

Navigation

-

Contents:

+

Documentation Content:

diff --git a/docs/build/html/_modules/index.html b/docs/build/html/_modules/index.html index 8396f91..6473c82 100644 --- a/docs/build/html/_modules/index.html +++ b/docs/build/html/_modules/index.html @@ -33,6 +33,8 @@

All modules for which code is available

Navigation

-

Contents:

+

Documentation Content:

diff --git a/docs/build/html/_modules/lark/tree.html b/docs/build/html/_modules/lark/tree.html new file mode 100644 index 0000000..8207153 --- /dev/null +++ b/docs/build/html/_modules/lark/tree.html @@ -0,0 +1,373 @@ + + + + + + + lark.tree — CEL in Python documentation + + + + + + + + + + + + + + + + + + +
+
+
+ + +
+ +

Source code for lark.tree

+import sys
+from copy import deepcopy
+
+from typing import List, Callable, Iterator, Union, Optional, Generic, TypeVar, TYPE_CHECKING
+
+if TYPE_CHECKING:
+    from .lexer import TerminalDef, Token
+    try:
+        import rich
+    except ImportError:
+        pass
+    from typing import Literal
+
+###{standalone
+
+class Meta:
+
+    empty: bool
+    line: int
+    column: int
+    start_pos: int
+    end_line: int
+    end_column: int
+    end_pos: int
+    orig_expansion: 'List[TerminalDef]'
+    match_tree: bool
+
+    def __init__(self):
+        self.empty = True
+
+
+_Leaf_T = TypeVar("_Leaf_T")
+Branch = Union[_Leaf_T, 'Tree[_Leaf_T]']
+
+
+class Tree(Generic[_Leaf_T]):
+    """The main tree class.
+
+    Creates a new tree, and stores "data" and "children" in attributes of the same name.
+    Trees can be hashed and compared.
+
+    Parameters:
+        data: The name of the rule or alias
+        children: List of matched sub-rules and terminals
+        meta: Line & Column numbers (if ``propagate_positions`` is enabled).
+            meta attributes: (line, column, end_line, end_column, start_pos, end_pos,
+                              container_line, container_column, container_end_line, container_end_column)
+            container_* attributes consider all symbols, including those that have been inlined in the tree.
+            For example, in the rule 'a: _A B _C', the regular attributes will mark the start and end of B,
+            but the container_* attributes will also include _A and _C in the range. However, rules that
+            contain 'a' will consider it in full, including _A and _C for all attributes.
+    """
+
+    data: str
+    children: 'List[Branch[_Leaf_T]]'
+
+    def __init__(self, data: str, children: 'List[Branch[_Leaf_T]]', meta: Optional[Meta]=None) -> None:
+        self.data = data
+        self.children = children
+        self._meta = meta
+
+    @property
+    def meta(self) -> Meta:
+        if self._meta is None:
+            self._meta = Meta()
+        return self._meta
+
+    def __repr__(self):
+        return 'Tree(%r, %r)' % (self.data, self.children)
+
+    def _pretty_label(self):
+        return self.data
+
+    def _pretty(self, level, indent_str):
+        yield f'{indent_str*level}{self._pretty_label()}'
+        if len(self.children) == 1 and not isinstance(self.children[0], Tree):
+            yield f'\t{self.children[0]}\n'
+        else:
+            yield '\n'
+            for n in self.children:
+                if isinstance(n, Tree):
+                    yield from n._pretty(level+1, indent_str)
+                else:
+                    yield f'{indent_str*(level+1)}{n}\n'
+
+    def pretty(self, indent_str: str='  ') -> str:
+        """Returns an indented string representation of the tree.
+
+        Great for debugging.
+        """
+        return ''.join(self._pretty(0, indent_str))
+
+    def __rich__(self, parent:Optional['rich.tree.Tree']=None) -> 'rich.tree.Tree':
+        """Returns a tree widget for the 'rich' library.
+
+        Example:
+            ::
+                from rich import print
+                from lark import Tree
+
+                tree = Tree('root', ['node1', 'node2'])
+                print(tree)
+        """
+        return self._rich(parent)
+
+    def _rich(self, parent):
+        if parent:
+            tree = parent.add(f'[bold]{self.data}[/bold]')
+        else:
+            import rich.tree
+            tree = rich.tree.Tree(self.data)
+
+        for c in self.children:
+            if isinstance(c, Tree):
+                c._rich(tree)
+            else:
+                tree.add(f'[green]{c}[/green]')
+
+        return tree
+
+    def __eq__(self, other):
+        try:
+            return self.data == other.data and self.children == other.children
+        except AttributeError:
+            return False
+
+    def __ne__(self, other):
+        return not (self == other)
+
+    def __hash__(self) -> int:
+        return hash((self.data, tuple(self.children)))
+
+    def iter_subtrees(self) -> 'Iterator[Tree[_Leaf_T]]':
+        """Depth-first iteration.
+
+        Iterates over all the subtrees, never returning to the same node twice (Lark's parse-tree is actually a DAG).
+        """
+        queue = [self]
+        subtrees = dict()
+        for subtree in queue:
+            subtrees[id(subtree)] = subtree
+            queue += [c for c in reversed(subtree.children)
+                      if isinstance(c, Tree) and id(c) not in subtrees]
+
+        del queue
+        return reversed(list(subtrees.values()))
+
+    def iter_subtrees_topdown(self):
+        """Breadth-first iteration.
+
+        Iterates over all the subtrees, return nodes in order like pretty() does.
+        """
+        stack = [self]
+        stack_append = stack.append
+        stack_pop = stack.pop
+        while stack:
+            node = stack_pop()
+            if not isinstance(node, Tree):
+                continue
+            yield node
+            for child in reversed(node.children):
+                stack_append(child)
+
+    def find_pred(self, pred: 'Callable[[Tree[_Leaf_T]], bool]') -> 'Iterator[Tree[_Leaf_T]]':
+        """Returns all nodes of the tree that evaluate pred(node) as true."""
+        return filter(pred, self.iter_subtrees())
+
+    def find_data(self, data: str) -> 'Iterator[Tree[_Leaf_T]]':
+        """Returns all nodes of the tree whose data equals the given data."""
+        return self.find_pred(lambda t: t.data == data)
+
+###}
+
+    def expand_kids_by_data(self, *data_values):
+        """Expand (inline) children with any of the given data values. Returns True if anything changed"""
+        changed = False
+        for i in range(len(self.children)-1, -1, -1):
+            child = self.children[i]
+            if isinstance(child, Tree) and child.data in data_values:
+                self.children[i:i+1] = child.children
+                changed = True
+        return changed
+
+
+    def scan_values(self, pred: 'Callable[[Branch[_Leaf_T]], bool]') -> Iterator[_Leaf_T]:
+        """Return all values in the tree that evaluate pred(value) as true.
+
+        This can be used to find all the tokens in the tree.
+
+        Example:
+            >>> all_tokens = tree.scan_values(lambda v: isinstance(v, Token))
+        """
+        for c in self.children:
+            if isinstance(c, Tree):
+                for t in c.scan_values(pred):
+                    yield t
+            else:
+                if pred(c):
+                    yield c
+
+    def __deepcopy__(self, memo):
+        return type(self)(self.data, deepcopy(self.children, memo), meta=self._meta)
+
+    def copy(self) -> 'Tree[_Leaf_T]':
+        return type(self)(self.data, self.children)
+
+    def set(self, data: str, children: 'List[Branch[_Leaf_T]]') -> None:
+        self.data = data
+        self.children = children
+
+
+ParseTree = Tree['Token']
+
+
+class SlottedTree(Tree):
+    __slots__ = 'data', 'children', 'rule', '_meta'
+
+
+def pydot__tree_to_png(tree: Tree, filename: str, rankdir: 'Literal["TB", "LR", "BT", "RL"]'="LR", **kwargs) -> None:
+    graph = pydot__tree_to_graph(tree, rankdir, **kwargs)
+    graph.write_png(filename)
+
+
+def pydot__tree_to_dot(tree: Tree, filename, rankdir="LR", **kwargs):
+    graph = pydot__tree_to_graph(tree, rankdir, **kwargs)
+    graph.write(filename)
+
+
+def pydot__tree_to_graph(tree: Tree, rankdir="LR", **kwargs):
+    """Creates a colorful image that represents the tree (data+children, without meta)
+
+    Possible values for `rankdir` are "TB", "LR", "BT", "RL", corresponding to
+    directed graphs drawn from top to bottom, from left to right, from bottom to
+    top, and from right to left, respectively.
+
+    `kwargs` can be any graph attribute (e. g. `dpi=200`). For a list of
+    possible attributes, see https://www.graphviz.org/doc/info/attrs.html.
+    """
+
+    import pydot  # type: ignore[import-not-found]
+    graph = pydot.Dot(graph_type='digraph', rankdir=rankdir, **kwargs)
+
+    i = [0]
+
+    def new_leaf(leaf):
+        node = pydot.Node(i[0], label=repr(leaf))
+        i[0] += 1
+        graph.add_node(node)
+        return node
+
+    def _to_pydot(subtree):
+        color = hash(subtree.data) & 0xffffff
+        color |= 0x808080
+
+        subnodes = [_to_pydot(child) if isinstance(child, Tree) else new_leaf(child)
+                    for child in subtree.children]
+        node = pydot.Node(i[0], style="filled", fillcolor="#%x" % color, label=subtree.data)
+        i[0] += 1
+        graph.add_node(node)
+
+        for subnode in subnodes:
+            graph.add_edge(pydot.Edge(node, subnode))
+
+        return node
+
+    _to_pydot(tree)
+    return graph
+
+ +
+ +
+
+ +
+
+ + + + + + + \ No newline at end of file diff --git a/docs/build/html/_plantuml/01/01f21512c6b94e19efccb4676b518bed63780ddd.png b/docs/build/html/_plantuml/01/01f21512c6b94e19efccb4676b518bed63780ddd.png new file mode 100644 index 0000000..080aa3a Binary files /dev/null and b/docs/build/html/_plantuml/01/01f21512c6b94e19efccb4676b518bed63780ddd.png differ diff --git a/docs/build/html/_plantuml/16/165a23eb11f806bdd308ffac47a78125152ade58.png b/docs/build/html/_plantuml/16/165a23eb11f806bdd308ffac47a78125152ade58.png new file mode 100644 index 0000000..b02b306 Binary files /dev/null and b/docs/build/html/_plantuml/16/165a23eb11f806bdd308ffac47a78125152ade58.png differ diff --git a/docs/build/html/_plantuml/59/59894b154086c2a2ebe8ae46d280279ecce7cf8f.png b/docs/build/html/_plantuml/59/59894b154086c2a2ebe8ae46d280279ecce7cf8f.png new file mode 100644 index 0000000..16f9a9c Binary files /dev/null and b/docs/build/html/_plantuml/59/59894b154086c2a2ebe8ae46d280279ecce7cf8f.png differ diff --git a/docs/build/html/_plantuml/60/60c95188891b5d1267db67bb61a17e9fd7c08060.png b/docs/build/html/_plantuml/60/60c95188891b5d1267db67bb61a17e9fd7c08060.png new file mode 100644 index 0000000..cb501f6 Binary files /dev/null and b/docs/build/html/_plantuml/60/60c95188891b5d1267db67bb61a17e9fd7c08060.png differ diff --git a/docs/build/html/_plantuml/60/60c95188891b5d1267db67bb61a17e9fd7c08060.png.newa1v5axm0 b/docs/build/html/_plantuml/60/60c95188891b5d1267db67bb61a17e9fd7c08060.png.newa1v5axm0 new file mode 100644 index 0000000..e69de29 diff --git a/docs/build/html/_plantuml/60/60c95188891b5d1267db67bb61a17e9fd7c08060.png.newlpv3pfrv b/docs/build/html/_plantuml/60/60c95188891b5d1267db67bb61a17e9fd7c08060.png.newlpv3pfrv new file mode 100644 index 0000000..e69de29 diff --git a/docs/build/html/_plantuml/60/60c95188891b5d1267db67bb61a17e9fd7c08060.png.newv2qg6lg_ b/docs/build/html/_plantuml/60/60c95188891b5d1267db67bb61a17e9fd7c08060.png.newv2qg6lg_ new file mode 100644 index 0000000..e69de29 diff --git a/docs/build/html/_plantuml/a8/a8a77cea7a1f2555dbb1cb6cea10dc21777c76bf.png b/docs/build/html/_plantuml/a8/a8a77cea7a1f2555dbb1cb6cea10dc21777c76bf.png new file mode 100644 index 0000000..96c3652 Binary files /dev/null and b/docs/build/html/_plantuml/a8/a8a77cea7a1f2555dbb1cb6cea10dc21777c76bf.png differ diff --git a/docs/build/html/_plantuml/a8/a8a77cea7a1f2555dbb1cb6cea10dc21777c76bf.png.new1cmjxeth b/docs/build/html/_plantuml/a8/a8a77cea7a1f2555dbb1cb6cea10dc21777c76bf.png.new1cmjxeth new file mode 100644 index 0000000..e69de29 diff --git a/docs/build/html/_plantuml/a8/a8a77cea7a1f2555dbb1cb6cea10dc21777c76bf.png.new4bry_4gw b/docs/build/html/_plantuml/a8/a8a77cea7a1f2555dbb1cb6cea10dc21777c76bf.png.new4bry_4gw new file mode 100644 index 0000000..e69de29 diff --git a/docs/build/html/_plantuml/a8/a8a77cea7a1f2555dbb1cb6cea10dc21777c76bf.png.new58drlmt5 b/docs/build/html/_plantuml/a8/a8a77cea7a1f2555dbb1cb6cea10dc21777c76bf.png.new58drlmt5 new file mode 100644 index 0000000..e69de29 diff --git a/docs/build/html/_plantuml/ac/ac649ac296fc6bea0cee4f6cb2d92533640e6763.png b/docs/build/html/_plantuml/ac/ac649ac296fc6bea0cee4f6cb2d92533640e6763.png new file mode 100644 index 0000000..391d74a Binary files /dev/null and b/docs/build/html/_plantuml/ac/ac649ac296fc6bea0cee4f6cb2d92533640e6763.png differ diff --git a/docs/build/html/_plantuml/c9/c970d97dc7e0a41eb7666fff5d466440fc8d67cf.png b/docs/build/html/_plantuml/c9/c970d97dc7e0a41eb7666fff5d466440fc8d67cf.png new file mode 100644 index 0000000..75c2ffd Binary files /dev/null and b/docs/build/html/_plantuml/c9/c970d97dc7e0a41eb7666fff5d466440fc8d67cf.png differ diff --git a/docs/build/html/_sources/api.rst.txt b/docs/build/html/_sources/api.rst.txt index da47f98..7515b4f 100644 --- a/docs/build/html/_sources/api.rst.txt +++ b/docs/build/html/_sources/api.rst.txt @@ -5,11 +5,30 @@ .. _`api`: ########### -CEL-Py API +API ########### Details of the CEL-Python implementation and the API to the various components. +The following modules are described here: + +- celpy_ (The ``__init__.py`` file defines this.) + +- \_\_main\_\_ (The CLI main application.) + +- adapter_ + +- c7nlib_ + +- celparser_ + +- celtypes_ + +- evaluation_ + +``celpy`` +========= + .. automodule:: celpy.__init__ ``__main__`` @@ -17,18 +36,28 @@ Details of the CEL-Python implementation and the API to the various components. .. automodule:: celpy.__main__ -celtypes -========= +``adapter`` +=========== -.. automodule:: celpy.celtypes +.. automodule:: celpy.adapter -evaluation -========== +``c7nlib`` +=========== -.. automodule:: celpy.evaluation +.. automodule:: celpy.c7nlib -parser -====== +``celparser`` +============= .. automodule:: celpy.celparser +``celtypes`` +============= + +.. automodule:: celpy.celtypes + +``evaluation`` +============== + +.. automodule:: celpy.evaluation + diff --git a/docs/build/html/_sources/c7n_functions.rst.txt b/docs/build/html/_sources/c7n_functions.rst.txt index 62b8c6d..edc12bd 100644 --- a/docs/build/html/_sources/c7n_functions.rst.txt +++ b/docs/build/html/_sources/c7n_functions.rst.txt @@ -5,15 +5,13 @@ C7N Functions Required ###################### -This survey of C7N filter clauses is based on source code and -on an analysis of working policies. The required functions -are grouped into four clusters, depending on the presence of absence of -the "op" operator clause, and the number of resource types using the feature. +This survey of C7N filter clauses is based on source code and on an analysis of working policies. +The required functions are grouped into four clusters, depending on the presence of absence of the "op" operator clause, and the number of resource types using the feature. Within each group, they are ranked in order of popularity. For each individual type of filter clause, we provide the following details: -- The C7N sechema definition. +- The C7N schema definition. - The resource types where the filter type is used. @@ -43,7 +41,7 @@ interface. 1. Separate from C7N. The CEL processing is outside C7N, and capable of standing alone. CEL is focused on Protobuf (and JSON) objects. The interface to C7N is via the :mod:`c7nlib` library of functions. These do **not** depend - on imports from the C7N project, but rely on a `CELFilter` class offering specific methods. + on imports from the C7N project, but rely on a ``CELFilter`` class offering specific methods. Access to C7N objects and their associated methods is limited to the features exposed through the function library and the expected class definition. diff --git a/docs/build/html/_sources/cli.rst.txt b/docs/build/html/_sources/cli.rst.txt index 3cb4b02..96a5319 100644 --- a/docs/build/html/_sources/cli.rst.txt +++ b/docs/build/html/_sources/cli.rst.txt @@ -6,50 +6,302 @@ CLI Use of CEL-Python ###################### -We can read JSON directly from stdin, making this a bit like JQ. +While CEL-Python's primary use case is integration into an DSL-based application to provide expressions with a uniform syntax and well-defined semantics. +The expression processing capability is also available as a CLI implemented in the ``celpy`` package. + +SYNOPSIS +======== :: - % PYTHONPATH=src python -m celpy '.this.from.json * 3 + 3' <, --arg + + Define argument variables, types, and (optional) values. + If the argument value is omitted, then an environment variable will be examined to find the value. + For example, ``--arg HOME:string`` makes the :envvar:`HOME` environment variable's value available to the CEL expression. + +.. option:: -b, --boolean + + Return a status code value based on the boolean output. + + true has a status code of 0 + + false has a statis code of 1 + + Any exception has a stats code of 2 + +.. option:: -n, --null-input + + Do not read JSON input from stdin + +.. option:: -s, --slurp + + Treat all input as a single JSON document. + The default is to treat each line of input as a separate NLJSON document. + +.. option:: -i, --interactive + + Operate interactively from a ``CEL>`` prompt. + In :option:`-i` mode, the rest of the options are ignored. + +.. option:: -p, --json-package + + Each NDJSON input (or the single input in :option:`-s` mode) + is a CEL package. + +.. option:: -d, --json-document + + Each NDJSON input (or the single input in :option:`-s` mode) + is a separate CEL variable. + +.. option:: -f , --format + + Use Python formating instead of JSON conversion of results; + Example ``--format .6f`` to format a ``DoubleType`` result + +.. option:: expr + + A CEL expression to evaluate. + +DESCRIPTION +============ + +This provides shell-friendly expression processing. +It follows patterns from several programs. + +:jq: + The ``celpy`` application will read newline-delimited JSON + from stdin. + It can also read a single, multiline JSON document in ``--slurp`` mode. + + This will evaluate the expression for each JSON document. + + .. note:: + + ``jq`` uses ``.`` to refer the current document. By setting a package + name of ``"jq"`` with the :option:`-p` option, e.g., ``-p jq``, + and placing the JSON object in the same package, we achieve + similar syntax. + +:expr: + The ``celpy`` application does everything ``expr`` does, but the syntax is different. + + The output of comparisons in ``celpy`` is boolean, where by default. + The ``expr`` program returns an integer 1 or 0. + Use the :option:`-f` option, for example, ``-f 'd'`` to see decimal output instead of Boolean text values. + +:test: + This does what ``test`` does using CEL syntax. + The ``stat()`` function retrieves a mapping with various file status values. + + Use the :option:`-b` option to set the exit status code from the Boolean result. + + A ``true`` value becomes a 0 exit code. + + A ``false`` value becomes a 1 exit code. + +:bc: + THe little-used linux ``bc`` application has several complex function definitions and other programming support. + CEL can evaluate some ``bc``\\ -like expressions. + It could be extended to mimic ``bc``. + +Additionally, in :option:`--interactive` mode, +there's a REPL with a ``CEL>`` prompt. + +Arguments, Types, and Namespaces +--------------------------------- + +The :option:`--arg` options must provide a variable name and type. +CEL objects rely on the :py:mod:`celpy.celtypes` definitions. + +Because of the close association between CEL and protobuf, some well-known protobuf types +are also supported. + +The value for a variable is optional. +If it is not provided, then the variable is presumed to be an environment variable. +While many environment variables are strings, the type is still required. +For example, use ``--arg HOME:string`` to get the value of the :envvar:`HOME` environment variable. + +FILES +====== + +By default, JSON documents are read from stdin in NDJSON format (http://jsonlines.org/, http://ndjson.org/). +For each JSON document, the expression is evaluated with the document in a default +package. This allows `.name` to pick items from the document. + +By default, the output is JSON serialized. +This means strings will be JSON-ified and have quotes. +Using the :option:`-f` option will expect a single, primitive type that can be formatting using Python's string formatting mini-language. + +ENVIRONMENT VARIABLES +===================== + +Enhanced logging is available when :envvar:`CEL_TRACE` is defined. +This is quite voluminous; tracing most pieces of the AST during evaluation. + +CONFIGURATION +============= + +Logging configuration is read from the ``celpy.toml`` file. +See :ref:`configuration` for details. + +EXIT STATUS +=========== + +Normally, it's zero. + +When the :option:`-b` option is used then the final expression determines the status code. + +A value of ``true`` returns 0. + +A value of ``false`` returns 1. + +Other values or an evaluation error exception will return 2. + +EXAMPLES +======== + +We can read JSON directly from stdin, making this a bit like the **jq** application. +We provide a JQ expression, ``'.this.from.json * 3 + 3'``, and a JSON document. +The standard output is the computed result. + +.. code-block:: bash + + % python -m celpy '.this.from.json * 3 + 3' < {"this": {"from": {"json": 13}}} heredoc> EOF 42 +The default behavior is to read and process stdin, where each line is a separate JSON document. +This is the Newline-Delimited JSON format. +(See https://jsonlines.org and https://github.com/ndjson/ndjson-spec). + +The ``-s/--slurp`` treats the stdin as a single JSON document, spread over multiple lines. +This parallels the way the the **jq** application handles JSON input. + +We can avoid reading stdin by using the ``-n/--null-input`` option. +This option will evaluate the expression using only command-line argument values. + It's also a desk calculator. -:: +.. code-block:: bash % python -m celpy -n '355.0 / 113.0' 3.1415929203539825 -And, yes, this has a tiny advantage over ``python -c '355/113'``. Most notably, the ability -to embed Google CEL into other contexts where you don't *really* want Python's power. -There's no CEL ``import`` or built-in ``exec()`` function to raise concerns. +And, yes, this use case has a tiny advantage over ``python -c '355/113'``. +Most notably, the ability to embed Google CEL into other contexts where you don't *really* want Python's power. +There's no CEL ``import`` or built-in ``eval()`` function to raise security concerns. -We can provide a ``-d`` option to define objects with particular data types, like JSON. -This is particularly helpful for providing protobuf message definitions. +We can provide a ``-a/--arg`` option to define a name in the current activation with particular data type. +The expression, ``'x * 3 + 3'`` depends on a ``x`` variable, set by the ``-a`` option. +Note the ``variable:type`` syntax for setting the type of the variable. -:: +.. code-block:: bash - % PYTHONPATH=src python -m celpy -n -ax:int=13 'x * 3 + 3' + % python -m celpy -n -ax:int=13 'x * 3 + 3' 42 -This command sets a variable ``x`` then evaluates the expression. And yes, this is what -``expr`` does. CEL can do more. For example, floating-point math. +This is what the bash ``expr`` command does. +CEL can do more. +For example, floating-point math. +Here we've set two variables, ``x`` and ``tot``, before evaluating an expression. -:: +.. code-block:: bash - % PYTHONPATH=src python -m celpy -n -ax:double=113 -atot:double=355 '100. * x/tot' + % python -m celpy -n -ax:double=113 -atot:double=355 '100. * x/tot' 31.830985915492956 -We can also mimic the ``test`` command. +If you omit the ``=`` from the ``-a`` option, then an environment variable's +value will be bound to the variable name in the activation. -:: +.. code-block:: bash + + % TOTAL=41 python -m celpy -n -aTOTAL:int 'TOTAL + 1' + 42 + +Since these operations involves explict type conversions, be aware of the possibility of syntax error exceptions. + +.. code-block:: bash - % PYTHONPATH=src python -m celpy -n -ax:int=113 -atot:int=355 -b 'x > tot' + % TOTAL="not a number" python -m celpy -n -aTOTAL:int 'TOTAL + 1' + usage: celpy [-h] [-v] [-a ARG] [-n] [-s] [-i] [--json-package NAME] [--json-document NAME] [-b] [-f FORMAT] [expr] + celpy: error: argument -a/--arg: arg TOTAL:int value invalid for the supplied type + + + +We can also use this instead of the bash ``test`` command. +We can bind values with the ``-a`` options and then compare them. +The ``-b/--boolean`` option sets the status value based on the boolean result value. +The output string is the CEL literal value ``false``. +The status code is a "failure" code of 1. + +.. code-block:: bash + + % python -m celpy -n -ax:int=113 -atot:int=355 -b 'x > tot' false % echo $? 1 -The intent is to provide a common implementation for aritmetic and logic. +Here's another example that shows the ``stat()`` function to get filesystem status. + +.. code-block:: bash + + % python -m celpy -n -aHOME 'HOME.stat()' + {"st_atime": "2025-07-06T20:27:21Z", "st_birthtime": "2006-11-27T18:30:03Z", "st_ctime": "2025-07-06T20:27:20Z", "st_dev": 16777234, "st_ino": 341035, "st_mtime": "2025-07-06T20:27:20Z", "st_nlink": 135, "st_size": 4320, "group_access": true, "user_access": true, "kind": "d", "setuid": false, "setgid": false, "sticky": false, "r": true, "w": true, "x": true, "st_blksize": 4096, "st_blocks": 0, "st_flags": 0, "st_rdev": 0, "st_gen": 0} + +As an example, to compare modification time between two files, use an expression like ``f1.stat().st_mtime < f2.stat().st_mtime``. + +This is longer than the traditional bash expression, but much more clear. + +The file "kind" is a one-letter code: +:b: block +:c: character-mode +:d: directory +:f: regular file +:p: FIFO or pipe +:l: symbolic link +:s: socket + +The ``r``, ``w``, and ``x`` attributes indicate if the current effective userid can read, write, or execute the file. This comes from the detailed permission bits. + +The intent is to provide a single, uniform implementation for arithmetic and logic operations. +The primary use case integration into an DSL-based application to provide expressions without the mental burden of writing the parser and evaluator. + +We can also use CEL interactively, because, why not? + +.. code-block:: bash + + % python -m celpy -i + Enter an expression to have it evaluated. + CEL> 355. / 113. + 3.1415929203539825 + CEL> ? + + Documented commands (type help ): + ======================================== + bye exit help quit set show + + CEL> help set + Set variable expression + + Evaluates the expression, saves the result as the given variable in the current activation. + + CEL> set a 6 + 6 + CEL> set b 7 + 7 + CEL> a * b + 42 + CEL> show + {'a': IntType(6), 'b': IntType(7)} + CEL> bye + % + +The ``bye``, ``exit``, and ``quit`` commands all exit the application. diff --git a/docs/build/html/_sources/configuration.rst.txt b/docs/build/html/_sources/configuration.rst.txt index e029b5d..93bf760 100644 --- a/docs/build/html/_sources/configuration.rst.txt +++ b/docs/build/html/_sources/configuration.rst.txt @@ -2,25 +2,13 @@ # Copyright 2020 The Cloud Custodian Authors. # SPDX-License-Identifier: Apache-2.0 +.. _configuration: + ###################### Configuration ###################### -The CLI application can bind argument values from the environment. -The command-line provides variable names and type information. -The OS environment provides string values. - -.. code:: bash - - export x=6 - export y=7 - celpy -n --arg x:int --arg y:int 'x*y' - 42 - -While this example uses the OS environment, -it isn't the usual sense of *configuration*. -The only configuration options available for the command-line application are the logging configuration. - +The **celpy** package uses a configuration file to set the logging options. If a ``celpy.toml`` file exists in the local directory or the user's ``HOME`` directory, this will be used to provide logging configuration for the ``celpy`` application. This file must have a ``logging`` paragraph. @@ -40,11 +28,41 @@ This paragraph can contain the parameters for logging configuration. class = "logging.StreamHandler" formatter = "console" +This provides minimal log output, showing only warnings, errors, and fatal error messages. +The ``root.level`` needs to be "INFO" or "DEBUG" to see more output. +Setting a specific logger's level to "DEBUG" will raise the logging level for a specific component. + +All of the **celpy** loggers have names starting with ``celpy.``. +This permits integration with other application without polluting those logs with **celpy** output. + To enable very detailed debugging, do the following: - Set the ``CEL_TRACE`` environment variable to some non-empty value, like ``"true"``. This enables a ``@trace`` decorator on some evaluation methods. -- In a ``[logging.loggers.celpy.Evaluator]`` paragraph, set ``level = "DEBUG"``. +- Add a ``[logging.loggers.celpy.Evaluator]`` paragraph, with ``level = "DEBUG"``. + This can be done for any of the ``celpy`` components with loggers. + +- In the ``[logging]`` paragraph, set ``root.level = "DEBUG"``. + +Loggers include the following: + +- ``celpy`` + +- ``celpy.Runner`` + +- ``celpy.Environment`` + +- ``celpy.repl`` + +- ``celpy.c7nlib`` + +- ``celpy.celtypes`` + +- ``celpy.evaluation`` + +- ``celpy.NameContainer`` + +- ``celpy.Evaluator`` -- Set the ``[logging]`` paragraph, set ``root.level = "DEBUG"``. +- ``celpy.Transpiler`` diff --git a/docs/build/html/_sources/development.rst.txt b/docs/build/html/_sources/development.rst.txt new file mode 100644 index 0000000..02a00f3 --- /dev/null +++ b/docs/build/html/_sources/development.rst.txt @@ -0,0 +1,288 @@ +.. comment + # Copyright 2025 The Cloud Custodian Authors. + # SPDX-License-Identifier: Apache-2.0 + +###################### +Development Tools +###################### + +The development effort is dependent on several parts of the CEL project. + +1. The language specification. https://github.com/google/cel-spec/blob/master/doc/langdef.md + +2. The test cases. https://github.com/google/cel-spec/tree/master/tests/simple/testdata + +The language specification is transformed into a Lark grammar. +This is in the ``src/cel.lark`` file. +This changes very slowly. +Any changes must be reflected (manually) by revising the lark version of the EBNF. + +The test cases present a more challenging problem. + +A tool, ``pb2g.py``, converts the test cases from Protobuf messages to Gherkin scenarios. + +.. uml:: + + @startuml + + file source as "source protobuf test cases" + file features as "Gherkin feature files" + + source --> [pb2g.py] + [pb2g.py] --> [Docker] : "Uses" + [Docker] ..> [mkgherkin.go] : "Runs" + [pb2g.py] --> features + @enduml + + +The pb2g Tool +============== + +The ``pb2g.py`` Python application converts a protobuf test case collection into a Gherkin Feature file. +These can be used to update the ``features`` directory. + +SYNOPSIS +--------- + +.. program:: python tools/pb2g.py [-g docker|local] [-o output] [-sv] source + +.. option:: -g , --gherkinizer , --output + + Where to write the feature file. + Generally, it's helpful to have the ``.textproto`` and ``.feature`` stems match. + The ``Makefile`` assures this. + +.. option:: -s, --silent + + No console output is produced + +.. option:: -v, --verbose + + Verbose debugging output on the console. + +.. option:: source + + A source ``.textproto`` file. + This is often the path to a file in a local download of https://github.com/google/cel-spec/tree/master/tests/simple/testdata. + + A URL for the source is **not** supported. + + +DESCRIPTION +----------- + +Convert one ``.textproto`` file to a Gherkin ``.feature`` file. +There are two steps to the conversion: + +1. Rewrite the ``.textproto`` into JSON. + This relies on common Go libraries, and is little more than a syntactic conversion. + +2. Rewrite the JSON copy of the ``.textproto`` into Gherkin. + This a little more fraught with special cases and exceptions. + The ``.textproto`` semantics can be confusing. + +FILES +----- + +:source: + A ``.textproto`` test case file from the CEL-spec repository. + +:output: + A ``.feature`` file with the same stem as the source file is written to the output directory. + ``basic.textproto`` will create ``basic.feature``. + +:interim: + An interim JSON-format file is created and deleted. + These are only visible in the event of a fatal error creating the Gherkin output. + +EXAMPLES +-------- + +The ``basic.textproto`` starts like this: + +.. code-block:: protobuf + + name: "basic" + description: "Basic conformance tests that all implementations should pass." + section { + name: "self_eval_zeroish" + description: "Simple self-evaluating forms to zero-ish values." + test { + name: "self_eval_int_zero" + expr: "0" + value: { int64_value: 0 } + } + test { + name: "self_eval_uint_zero" + expr: "0u" + value: { uint64_value: 0 } + } + +The Feature file created looks like this: + +.. code-block:: gherkin + + Feature: basic + Basic conformance tests that all implementations should pass. + + # self_eval_zeroish -- Simple self-evaluating forms to zero-ish values. + + Scenario: self_eval_int_zero + + When CEL expression "0" is evaluated + # int64_value:0 + Then value is IntType(source=0) + + + Scenario: self_eval_uint_zero + + When CEL expression "0u" is evaluated + # uint64_value:0 + Then value is UintType(source=0) + +The source files have a "section" heading which doesn't have a precise parallel in the Gherkin language. +The sections become comments in the Feature file. + +The ``features/steps`` directory has step definition modules that implement the ``Given``, ``When``, and ``Then`` clauses. + +.. py:module:: features.steps.c7n_integration + + Provides step definitions for the ``c7n_interface.feature``. + This is not part of the CEL language specification. + +.. py:module:: features.steps.cli_binding + + Provides step definitions for the ``expr_test_bc.feature``, ``json_query.feature``, neither of which are part of the CEL language specificaiton. + +.. py:module:: features.steps.integration_binding + + Provides step definitions for the features generated by the ``pb2g.py`` tool. + +The ``features/Makefile`` +========================= + +This Makefile has the following targets: + +:%.textproto: + This copies textproto files from the source directory + to the ``features`` directory. + The source is defined by the :envvar:`CEL_SIMPLE_TESTDATA` environment variable. + This will overwrite out-of-date files in the ``features`` directory. + + It's important to use **git** wisely and start with a clean branch of the project so changes can be rolled back. + +:%.feature: + This creates the ``.feature`` file from the ``.textproto`` file. + +:scan: + This phony target reads **all** of the ``.textproto`` files to be sure they can be converted to Gherkin. + If it concludes with the output ``"All files scanned successfully"``, then there are no surprising or unexpected features in the ``.textproto`` files. + +:clean-broken: + This phony target removes empty ``.feature`` files that may be left over when the conversion process crashes with a fatal error. + +:clean-features: + This phony target removes all of the ``.textproto``\ -based ``.feature`` files. + Manually created ``.feature`` files are left intact. + +:clean: + This phony target removes all ``.textproto`` and ``.feature`` files that are built from the CEL specification. + Manually created ``.feature`` files are left intact. + +Currently, the following feature files are built from the CEL specification. + +.. code-block:: bash + + basic.feature + comparisons.feature + conversions.feature + dynamic.feature + enums.feature + fields.feature + fp_math.feature + integer_math.feature + lists.feature + logic.feature + macros.feature + namespace.feature + parse.feature + plumbing.feature + proto2.feature + proto3.feature + string.feature + timestamps.feature + unknowns.feature + +The ``docs/Makefile`` +===================== + +This is a Sphinx ``Makefile`` to build documentation. +For more information, see https://www.sphinx-doc.org/en/master/index.html + +The Project ``Makefile`` +========================= + +A top-level Makefile has a number of phony targets: + +:build: + Runs ``uv build`` to create a distribution kit. + +:install-tools: + Pulls a ``golang`` Docker image and builds the ``mkgherkin`` image. + +:test: + Runs the Python 3.12 test environment to execute a quick test. + +:test-all: + Update the ``features`` files and run the full test suite. + +:test-wip: + Update the ``features`` files and run the WIP test environment -- these are tests flagged with @WIP markers. + +:test-tools: + Run a test of only the tools, then scan the ``features`` files to be sure they're still valid after the tool change. + +:docs: + Build the HTML documentation. + +:lint: + Runs the ``lint`` test environment to get code coverage, type hint checking, and other lint checks. + +:coverage: + Reproduce the most recent coverage report. + +:clean: + Remove a number of directories and their files: + + - ``.tox`` + + - ``.Python`` + + - ``bin`` + + - ``include`` + + - ``lib`` + + - ``pip-selfcheck`` + + - ``.json`` + +:benchmarks: + Run the applications in the ``benches`` directory to gather performance benchmark data. + + - ``large_resource_set.py`` + + - ``complex_expression.py`` diff --git a/docs/build/html/_sources/index.rst.txt b/docs/build/html/_sources/index.rst.txt index 1e548d7..5e3ab46 100644 --- a/docs/build/html/_sources/index.rst.txt +++ b/docs/build/html/_sources/index.rst.txt @@ -25,26 +25,33 @@ Pure Python implementation of Google Common Expression Language, https://opensou This implementation has minimal dependencies, runs quickly, and can be embedded into Python-based applications. Specifically, one intent is to be part of Cloud Custodian (C7N) as part of the security policy filter. -Interested in the API? There are three interesting topics: - -- :ref:`integration` -- :ref:`api` -- :ref:`data_structures` - -The integration into another application isn't a trivial ``import``. - .. toctree:: :maxdepth: 2 - :caption: Contents: + :caption: Documentation Content: installation cli configuration integration - api structure + api + development c7n_functions +Integration Overview +==================== + +Interested in the API for using this package? There are three key topics: + +- :ref:`integration` +- :ref:`api` +- :ref:`data_structures` + +The integration into another application is often a bit more than an ``import``. +This is because it involves combining CEL into another DSL. + +The current implementation includes Cloud Custodian (C7N) integration. + Indices and tables ================== diff --git a/docs/build/html/_sources/installation.rst.txt b/docs/build/html/_sources/installation.rst.txt index 42edddc..655b4ca 100644 --- a/docs/build/html/_sources/installation.rst.txt +++ b/docs/build/html/_sources/installation.rst.txt @@ -30,6 +30,7 @@ The optional RE2 package significantly speeds up regular expression matching. python -m pip install cel-python[re2] + .. warning:: In the case where the platform is "darwin" and the architecture is "arm64" and python is "3.13", diff --git a/docs/build/html/_sources/integration.rst.txt b/docs/build/html/_sources/integration.rst.txt index 86ef221..c36bfaa 100644 --- a/docs/build/html/_sources/integration.rst.txt +++ b/docs/build/html/_sources/integration.rst.txt @@ -8,32 +8,26 @@ Application Integration ######################## -We'll look at the essential base case for integration: -evaluate a function given some variable bindings. +We'll look at integration of CEL into another application from four perspectives: -Then we'll look at providing custom function bindings to extend -the environment. +1. We'll look at the essential base case for integration into another application. + This will use an ``Activation`` to provide values for variables. -We'll also look at additional examples from the Go implementation. -This will lead us to providing custom type providers -and custom type adapters. +2. A more sophisticated integration involves extending the environment with custom functions. + This can provide a well-defined interface between CEL expressions and application functionality. -There are a few exception and error-handling cases that are helpful -for writing CEL that tolerates certain kinds of data problems. +3. Some additional examples from the Go implementation show how extend the environment using new types. -Finally, we'll look at how CEL can be integrated into Cloud Custodian. +4. There are a few exception and error-handling cases that are part of working with Python types. -The Essentials -============== +5. Finally, we'll look at how CEL can be integrated into Cloud Custodian (C7N). -Here are two examples of variable bindings - -README ------- +Integration Essentials +====================== -Here's the example taken from the README. +Here's an example of variable bindings taken from a ``README`` example: -The baseline implementation works like this:: +.. code-block:: python >>> import celpy >>> cel_source = """ @@ -46,36 +40,49 @@ The baseline implementation works like this:: >>> ast = env.compile(cel_source) >>> prgm = env.program(ast) - >>> activation = { + >>> context = { ... "account": celpy.json_to_cel({"balance": 500, "overdraftProtection": False}), ... "transaction": celpy.json_to_cel({"withdrawal": 600}) ... } - >>> result = prgm.evaluate(activation) + >>> result = prgm.evaluate(context) >>> result BoolType(False) -The :py:class:`celpy.Environment` can include type adapters and type providers. It's not clear -how these should be implemented in Python or if they're even necessary. +The ``cel_source`` is an expression to be evaluated. +This references variables with names like ``account``, and ``transaction``. + +All CEL evaluation uses an :py:class:`celpy.Environment` object. +The :py:class:`celpy.Environment` is used to provide type annotations for variables. +It can provide a few other properties, including an overall package name, sometimes needed when working with protobuf types. -The compile step creates a syntax tree, which is used to create a final program to evaluate. -Currently, there's a two-step process because we might want to optimize or transform the AST prior -to evaluation. +The :py:meth:`Environment.compile` method creates a abstract syntax tree from the CEL source. +This will be used to create a final program to evaluate. +This method can raise the :py:exc:`CELSyntaxError` exception. -The activation provides specific variable values used to evaluate the program. +The :py:meth:`Environment.program` method creates a runner out of an abstract syntax tree. -To an extent, the Python classes are loosely based on the object model in https://github.com/google/cel-go. +Compiling and building a program is a two-step process to permit optimization or some other transformation the AST prior to evaluation. +The Lark parser (https://lark-parser.readthedocs.io/en/latest/classes.html) is used, and transformers are a first-class feature of this parser. + +The ``context`` mapping defines variables and provides their values. +This is used to evaluate the resulting program object. +The program will produce a value defined in the :py:mod:`celpy.celtypes` module. +In this example, it's a :py:mod:`celpy.celtypes.BoolType` value. + +The CEL types are all specializations of the obvious Python base types. +To an extent, these Python classes are partially based on the object model in https://github.com/google/cel-go. We don't need all the Go formalisms, however, and rely on Pythonic variants. -Simple example using builtin operators +Simple example using builtin types --------------------------------------- Here's an example taken from -https://github.com/google/cel-go/blob/master/examples/README.md +https://github.com/google/cel-go/blob/master/examples/README.md. +This will evaluate the CEL expression ``"Hello world! I'm " + name + "."`` with ``"CEL"`` passed as the ``name`` variable. -Evaluate expression ``"Hello world! I'm " + name + "."`` with ``CEL`` passed as -the ``name`` variable. +This is the original Go code: -.. code:: go +.. code-block:: go import ( "github.com/google/cel-go/cel" @@ -97,7 +104,9 @@ the ``name`` variable. fmt.Println(out) // Output:Hello world! I'm CEL. -Here's the Python version:: +Here's the Python version, following a similar outline: + +.. code-block:: python >>> import celpy >>> cel_source = """ @@ -109,59 +118,59 @@ Here's the Python version:: >>> ast = env.compile(cel_source) >>> prgm = env.program(ast) - >>> activation = { + >>> context = { ... "name": "CEL" ... } - >>> result = prgm.evaluate(activation) + >>> result = prgm.evaluate(context) >>> result "Hello world! I'm CEL." -There's a big open concern here: there's no formal type adapter implementation. -Nothing converts from the input value in the activation to the proper underlying -type. This relies on Python's built-in type conversions. +The steps include: -.. todo:: Handle type adapters properly. +1. Create a :py:class:`celpy.Environment` with annotations for any variables. + These kinds of type definitions are atypical for Python, but are part of the definition of the CEL language. -Function Bindings -================= +2. Use :py:meth:`celpy.Environment.compile` to create an AST. -Here are two more examples of binding, taken from -https://github.com/google/cel-go/blob/master/examples/README.md +3. Use :py:meth:`celpy.Environment.program` to build a :py:class:`celpy.Runner` object that will do the final evaluation. This includes the environment and the AST. -Note the complication here comes from the way the Go implementation resolves overloaded functions. -Each CEL overload of a function is described by a ``("name", [args], result)`` structure. -This allows for multiple type-specific overload versions of a generic function. +4. Use :py:meth:`celpy.Runner.evaluate` to evaluate the program with specific values for the defined variables. -The key of ``("name", [args], result)`` maps to a specific ``arg_name_arg()`` or ``name_arg()`` -overloaded implementation for specific argument types. +In the Go world, there's a formal type adapter to convert input values to the objects used by CEL. +For numerous types, a default adapter handles this. -For example, ``("greet", [StringType, StringType], StringType)`` maps to ``string_greet_string()``. +In Python, on the other hand, we define the type conversions as features of the Python versions of the CEL types. +This approach fits better with native Python programming. -This is emphatically not how Python generally works. A more Pythonic approach is to provide -a single, generic, function which examines the arguments and decides what to do. Python doesn't -generally do overloaded name resolution. -There are two choices: +Function Bindings +================= + +There are two function binding examples in +https://github.com/google/cel-go/blob/master/examples/README.md. -1. Build a mapping from ``("name", [args], result)`` to a specific overloaded implementation. - This pulls argument and result type coercion outside the Python function. - It matches the Go implementation, but can be confusing for Python implementers. - This requires exposing a great deal of machinery already available in a Python function - definition. +There is a complication here that based on the way the Go resolves overloaded functions. +In Go, each overload of a function is described by a ``("name", [args], result)`` data structure. +The key of ``("name", [args], result)`` maps to a specific ``arg_name_arg()`` or ``name_arg()`` overloaded implementation for specific argument types. +This allows for multiple type-specific overload versions of a generic function. -2. Ignore the complex type exposture techniques that Go requiees and dispatch to a Python function. - The Python function will sort out type variants and handle argument value coercion on its own. - This simplifies implementation down to name resolution. - Indeed, the type mapping rules can introspect Python's type annotations on the function - definition. +For example, a ``("greet", [StringType, StringType], StringType)`` structure is expected to map to a function ``string_greet_string()`` that has the expected signature. -We follow the 2nd alternative. The Python function binding relies -- exclusively -- on introspection -of the function provided. +This is emphatically not how Python generally works. +We follow a more Pythonic approach is to provide a single, generic, function which examines the arguments and decides what to do. +Outside type-checking, Python doesn't depend on overloaded name resolution. -Custom function on string type ------------------------------- +This means a Python function must then sort out type variants and handle argument value coercion on its own. +For most cases, the ``match/case`` statement is helpful for this. +The :py:func:`functools.singledispatch` decorator can also be helpful for this. -Evaluate expression ``i.greet(you)`` with: +The two examples have slightly different approaches to the CEL expression. +These are important in Go, but less important in Python. + +Custom function in Go +--------------------------------------- + +We want to evaluate the CEL expression ``i.greet(you)`` with: .. parsed-literal:: @@ -169,14 +178,14 @@ Evaluate expression ``i.greet(you)`` with: you -> world greet -> "Hello %s! Nice to meet you, I'm %s." +The idea here is the new ``greet()`` behaves like a method of a String. +The actual implementation, however, is not a method; it's a function of two arguments. -First we need to declare two string variables and `greet` function. -`NewInstanceOverload` must be used if we want to declare function which will -operate on a type. First element of slice passed as `argTypes` into -`NewInstanceOverload` is declaration of instance type. Next elements are -parameters of function. +First we need to declare two string variables and a ``greet()`` function. +In Go, a ``NewInstanceOverload`` must be used to provide annotations for variables and the function. +Here's the Go implementation: -.. code:: go +.. code-block:: go decls.NewVar("i", decls.String), decls.NewVar("you", decls.String), @@ -186,12 +195,15 @@ parameters of function. decls.String)) ... // Create env and compile +We've omitted the Go details of creating an environment and compiling the CEL expression. +These aren't different from the previous examples. -Let's implement `greet` function and pass it to `program`. We will be using -`Binary`, because `greet` function uses 2 parameters (1st instance, 2nd -function parameter). +Separately, a ``greetFunc()`` function must be defined. +In Go, this function is then bound to the ``"string_greet_string"`` overload, +ready for evaluation. +Here's the Go implementation: -.. code:: go +.. code-block:: go greetFunc := &functions.Overload{ Operator: "string_greet_string", @@ -208,7 +220,22 @@ function parameter). fmt.Println(out) // Output:Hello world! Nice to meet you, I'm CEL. -Here's the Python version:: +What's essential is defining some type information, then defining variables and functions that fit those types. + +The Python version has the same outline: + +1. An :py:class:`celpy.Environment` with type annotations for the two variables and the function. + +2. Compile the source. + +3. Define the ``greet()`` function. While the CEL syntax of ``i.greet(you)`` looks like a method +of the ``i`` variable's class, the function is simply has two positional parameters. + +4. Provide function implementation when creating the final :py:class:`celpy.Runner` instance. + +5. Evaluate the program with specific values for the two variables. + +.. code-block:: python >>> import celpy >>> cel_source = """ @@ -224,53 +251,36 @@ Here's the Python version:: >>> def greet(lhs: celpy.celtypes.StringType, rhs: celpy.celtypes.StringType) -> celpy.celtypes.StringType: ... return "Hello {1:s}! Nice to meet you, I'm {0:s}.\\n".format(lhs, rhs) >>> prgm = env.program(ast, functions=[greet]) - >>> activation = { + >>> context = { ... "i": "CEL", "you": "world" ... } - >>> result = prgm.evaluate(activation) + >>> result = prgm.evaluate(context) >>> result "Hello world! Nice to meet you, I'm CEL.\\n" -Define custom global function ------------------------------ - -Evaluate expression ``shake_hands(i,you)`` with: +The key concept here is to distinguish between three distinct attributes: -.. parsed-literal:: +1. Type annotations associated with variables or functions. - i -> CEL - you -> world - shake_hands -> "%s and %s are shaking hands." +2. The function implementations used to build the :py:class:`celpy.Runner`. + The method-like syntax of ``i.greet(you)`` is evaluated as ``greet(i, you)``. +3. The variable values, which provide a context in which the runner evaluates the CEL expression. -In order to declare global function we need to use `NewOverload`: +This reflects the idea that one CEL expression may be used to process data over and over again. -.. code:: go +Define custom global function +----------------------------- - decls.NewVar("i", decls.String), - decls.NewVar("you", decls.String), - decls.NewFunction("shake_hands", - decls.NewOverload("shake_hands_string_string", - []*exprpb.Type{decls.String, decls.String}, - decls.String)) - ... // Create env and compile. +In Go, this is a small, but important different.ce +We want to evaluate the expression ``shake_hands(i,you)``. +This uses a global function syntax instead of method syntax. - shakeFunc := &functions.Overload{ - Operator: "shake_hands_string_string", - Binary: func(lhs ref.Val, rhs ref.Val) ref.Val { - return types.String( - fmt.Sprintf("%s and %s are shaking hands.\n", lhs, rhs)) - }} - prg, err := env.Program(c, cel.Functions(shakeFunc)) +While Go has slight differences in how the function is defined, in Python, there is no change. - out, _, err := prg.Eval(map[string]interface{}{ - "i": "CEL", - "you": "world", - }) - fmt.Println(out) - // Output:CEL and world are shaking hands. +Here's the Python version: -Here's the Python version:: +.. code-block:: python >>> import celpy >>> cel_source = """ @@ -286,24 +296,25 @@ Here's the Python version:: >>> def shake_hands(lhs: celpy.celtypes.StringType, rhs: celpy.celtypes.StringType) -> celpy.celtypes.StringType: ... return f"{lhs} and {rhs} are shaking hands.\\n" >>> prgm = env.program(ast, functions=[shake_hands]) - >>> activation = { + >>> context = { ... "i": "CEL", "you": "world" ... } - >>> result = prgm.evaluate(activation) + >>> result = prgm.evaluate(context) >>> result 'CEL and world are shaking hands.\\n' +The ``shake_hands()`` function is essentially the same as the ``greet()`` function in the previous example. -For more examples of how to use CEL, see +For more examples of how to use CEL from Go, see https://github.com/google/cel-go/tree/master/cel/cel_test.go -Examples from Go implementation -================================ +More Examples from Go implementation +===================================== -See https://github.com/google/cel-go/blob/master/README.md +See https://github.com/google/cel-go/blob/master/README.md for five more examples. -.. code:: +.. code-block:: // Check whether a resource name starts with a group name. resource.name.startsWith("/groups/" + auth.claims.group) @@ -321,9 +332,10 @@ See https://github.com/google/cel-go/blob/master/README.md // in the JSON case. has(message.field) -Following one of the more complete examples through the README +Here's the first example, ``resource.name.startsWith("/groups/" + auth.claims.group)``. +The Go code is as follows: -.. code:: go +.. code-block:: go import( "github.com/google/cel-go/cel" @@ -353,7 +365,10 @@ Following one of the more complete examples through the README "group": "acme.co"}) fmt.Println(out) // 'true' -This has the following Python implementation:: +This has a Python implementation which is substantially similar. +Here's the Python code: + +.. code-block:: python >>> import celpy >>> decls = { @@ -363,60 +378,50 @@ This has the following Python implementation:: >>> env = celpy.Environment(annotations=decls) >>> ast = env.compile('name.startsWith("/groups/" + group)') >>> prgm = env.program(ast) - >>> activation = { + >>> context = { ... "name": "/groups/acme.co/documents/secret-stuff", ... "group": "acme.co", ... } - >>> result = prgm.evaluate(activation) + >>> result = prgm.evaluate(context) >>> result BoolType(True) +The general outline of compile, create a :py:class:`celpy.Runner`, and use :py:meth:`celpy.Runner.evaluate` to evaluate the CEL expression in a specific context is the central point here. + Exceptions and Errors ====================== -Exceptions raised in Python world will (eventually) crash the CEL evluation. -This gives the author of an extension function the complete traceback to help -fix the Python code. +Exceptions raised in Python world will (eventually) crash the CEL evaluation. +This gives the author of an extension function the complete traceback to help fix the Python code. No masking or rewriting of Python exceptions ever occurs in extension functions. -A special :exc:`celpy.EvalError` exception can be used in an extension function -to permit CEL's short-circuit logic processing to silence this exception. See the -https://github.com/google/cel-go/blob/master/README.md#partial-state for more examples -of how the short-circuit (partial state) operations work. +A special :py:exc:`celpy.CELEvalError` exception can be used in an extension function to permit CEL's short-circuit logic processing to check and ignore an exception. +See the https://github.com/google/cel-go/blob/master/README.md#partial-state for more examples of how the short-circuit (partial state) operations work. -An extension function must **return** a :exc:`celpy.EvalError` object -to allow processing to continue in spite of an uncomputable value. +An extension function can **return** a :py:exc:`celpy.CELEvalError` object instead of raising it. +This can allow processing to continue in spite of an uncomputable value. -:: +.. code-block:: python from celpy import * def my_extension(a: Value) -> Value: try: return celtypes.UintType(64 // a) except DivideByZeroError as ex: - return EvalError(f"my_extnsion({a}) error") - -The returned exception object allows short-circuit processing. For example, - -:: - - false && my_extension(0) + return CELEvalError(f"my_extension({a}) error") -This evaluates to ``false``. If computed, any :exc:`celpy.EvalError` object will be silently ignored. +The returned exception object allows short-circuit processing. +For example, the CEL expression ``false && my_extension(0)`` evaluates to ``false``. +If computed, any :exc:`celpy.CELEvalError` objects will be silently ignored because the short-circuit result is known from the presence of a ``false`` value. -On the other hand, - -:: - - true && my_extension(0) - -This will result in a visible :exc:`celpy.EvalError` result from the extension function. +On the other hand, the CEL expression ``true && my_extension(0)`` results in the :exc:`celpy.CELEvalError` result from the extension function. This will eventually be raised as an exception, so the framework using ``celpy`` can track this run-time error. -Cloud Custodian -=============== +Cloud Custodian (C7N) Integration +================================== Custodian Filters can be evaluated by CEL. +The idea is to extend the YAML-based DSL for policy documents to introduce easier-to-read expressions. As noted in https://github.com/cloud-custodian/cloud-custodian/issues/5759, a filter might look like the following:: @@ -432,15 +437,13 @@ following:: This replaces a complex sequence of nested ``- and:`` and ``- or:`` sub-documents with a CEL expression. -C7N processes resources by gathering resources, creating an instance of a subclass of the ``Filter`` -class, and evaluating an expression like ``take_action = list(filter(filter_instance, resource_list))``. +C7N processioning works by gathering resources, creating an instance of a subclass of the ``Filter`` class, and evaluating an expression like ``take_action = list(filter(filter_instance, resource_list))``. -The C7N filter expression in a given policy document is componsed of one or more atomic filter clauses, -combined by ``and``, ``or``, and ``not`` operators. +The C7N filter expression in a given policy document is composed of one or more atomic filter clauses, combined by ``and``, ``or``, and ``not`` operators. The filter as a whole is handled by the ``__call__()`` methods of subclasses of the ``BooleanGroupFilter`` class. Central to making this work is making the CEL expression into a function that can be applied to the ``resource`` object. -It appears that all CEL operations will need to have a number of values in their activations: +All CEL versions of a filter will need to have a the following two values in their activations: :resource: A :py:class:`celtypes.MapType` document with the resource details. @@ -448,14 +451,15 @@ It appears that all CEL operations will need to have a number of values in their :now: A :py:class:`celtypes.TimestampType` object with the current time. -Additional "global" objects may also be helpful. Baseline C7N Example -------------------- -The essence of the integration is to provide a resource to a function and receive a boolean result. +The essence of the integration is to provide a resource description to a function defined as a CEL expression, and receive a boolean result. + +Here's a base example: -Here's a base example:: +.. code-block:: python >>> import celpy >>> env = celpy.Environment() @@ -484,22 +488,26 @@ Here's a base example:: >>> prgm.evaluate(activation) BoolType(True) +In this case, the context contained only one variable, ``resource``. +It didn't require a definition of ``now``. + Bulk Filter Example ------------------- Pragmatically, C7N works via code somewhat like the following: -:: +.. code-block:: resources = [provider.describe(r) for r in provider.list(resource_type)] map(action, list(filter(cel_program, resources))) -An action is applied to those resources that pass some filter test. The filter looks for items not compliant -with policies. +An action is applied to those resources that pass some filter test. +Often, the action disables a resource to prevent data compromise. +The filter looks for items not compliant with policies so they can be deleted or disabled. The ``cel_program`` in the above example is an executable CEL program wrapped into a C7N ``Filter`` subclass. -:: +.. code-block:: >>> import celpy >>> import datetime @@ -537,23 +545,25 @@ The ``cel_program`` in the above example is an executable CEL program wrapped in >>> actionable [{'name': 'bad1', 'tags': {'not-owner': 'oops'}}, {'name': 'bad2', 'tags': {'owner': None}}] +For each resource, the ``tag_policy_filter`` object applied an internal ``self.prgm`` to the resource. +The internal ``self.prgm`` was built from the policy expression, stated in CEL. C7N Filter and Resource Types ------------------------------- -There are several parts to handling the various kinds of C7N filters in use. +The :py:mod:`celpy.c7nlib` module provides filter subclasses that include CEL processing. +There are two kinds of C7N filters in use. -1. The :py:mod:`c7n.filters` package defines about 23 generic filter classes, all of which need to - provide the ``resource`` object in the activation, and possibly provide a library of generic - CEL functions used for evaluation. - The general cases are of this is handled by the resource definition classes creating values in a JSON document. +1. The :py:mod:`c7n.filters` package defines about 23 generic filter classes. + These apply to a ``resource`` object. + Additionally, there's a library of generic functions used for evaluation. + Generally, the resource definition classes create values in a JSON document. These values reflect the state of the resource and any closely-related resources. 2. The :py:mod:`c7n.resources` package defines a number of additional resource-specific filters. - All of these, similarly, need to provide CEL values as part of the resource object. - These classes can also provide additional resource-specific CEL functions used for evaluation. + These classes can also provide additional resource-specific processing. -The atomic filter clauses have two general forms: +The atomic filter clauses within a policy document have two general forms: - Those with "op". These expose a resource attribute value, a filter comparison value, and an operator. @@ -566,8 +576,8 @@ The atomic filter clauses have two general forms: The breakdown of ``filter`` rules in the C7N policy schema has the following counts. .. csv-table:: - :header: category, count, notes + "('Common', 'Op')",21,"Used for more than one resource type, exposes resource details to CEL" "('Common', 'No-Op')",15,"Used for more than one resource type, does not expose resource details" "('Singleton', 'Op')",27,"Used for exactly one resource type, exposes resource details to CEL" diff --git a/docs/build/html/_sources/structure.rst.txt b/docs/build/html/_sources/structure.rst.txt index b2ee74c..5517636 100644 --- a/docs/build/html/_sources/structure.rst.txt +++ b/docs/build/html/_sources/structure.rst.txt @@ -4,51 +4,365 @@ .. _`data_structures`: -############### -Data Structures -############### +################################ +Architecture and Design +################################ -Run-Time -======== +We'll start with the C4 views: -An external client depends on the :py:class:`celpy.Environment`. +- `Context` -The :py:class:`celpy.Environment` builds the initial AST and the final runnable "program." -The :py:class:`celpy.Environment` may also contain a type provider and type adapters. +- `Container` -- this isn't too interesting, but it can help to see this. -The :py:class:`celpy.Environment` also builds -an :py:class:`celpy.evaluation.Activation` with the variable and function bindings -and the default package. +- `Components` -The :py:class:`celpy.evaluation.Activation` create a kind of chainmap for name -resolution. The chain has the following structure: + This is a collection of various design notes describing some implementation details. -- The end of the chain is the built-in defaults. + - `Compile-Time`_ -- A layer on top of this can be provided as part of integration into some other app or framework. + - `Evaluation-Time`_ -- The next layer is the "current" activation when evaluating a given expression. - This often has command-line variables. + - `CEL Types`_ + + - `Transpiler Missing Names`_ + + - `The member-dot Production`_ + +The code view is in the :ref:`api` section. + +Context +======= + +There are two distinct contexts for CEL Python: + +- The CLI -- as a stand-alone application. + +- As an importable module to provide expressions to a DSL. + +.. uml:: + + @startuml + skinparam actorStyle awesome + left to right direction + + package celpy { + package library { + usecase lib1 as "extend DSL with expressions" + usecase lib2 as "create program" + usecase lib3 as "evaluate program in context" + lib1 --> lib2 + lib1 --> lib3 + } + + package cli { + usecase cli1 as "**expr** features + --- + Use the -n option" + usecase cli2 as "**test** features + --- + Use the -nb options" + usecase cli3 as "**jq** features + Newline-Delimited or single JSON doc" + usecase cli4 as "interactive computation + --- + use the -i option" + } + + } + + actor app as "some app with a DSL" + app --> lib1 + + actor bash as "shell script" + bash --> cli1 + bash --> cli2 + bash --> cli3 + + actor user + user --> cli4 + + app <|- [c7n] + @enduml + +From the CLI, the ``celpy`` application has a number of use cases: + +- A shell script can use ``celpy`` as a command to replace other shell commands, including **expr**, **test**, and **jq**. + +- A person can run ``celpy`` interactively. + This allows experimentation. + It also supports exploring very complex JSON documents to understand their structure. + +As a library, an application (for example, C7N) can import ``celpy`` to provide an expression feature for the DSL. +This provides well-defined semantics, and widely-used syntax for the expression language. +There's an explicit separation between building a program and executing the program to allow caching an expression for multiple executions without the overhead of building a Lark parser or compiling the expression. + +Container +========= + +As a CLI, this is part of a shell script. It runs where the script runs. + +As a library, this is improted into the application to extend the DSL. + +There are no services offered or used. + +Components +========== + +The Python code base has a number of modules. + +- ``__init__`` -- the ``celpy`` package as a whole. + +- ``__main__`` -- the main applications used when running ``celpy``. + +- ``celparser`` -- a **Facade** for the Lark parser. + +- ``evaluation`` -- a **Facade** for run-time evaluation. + +- ``celtypes`` -- the underlying Python implementations of CEL data structures. + +- ``c7nlib``-- a collection of components the C7N can use to introduce CEL filters. + +- ``adapter`` -- Some JSON serialization components. + +Here's the conceptual organiation + +.. uml:: + + @startuml + + package celpy { + component "~__init__" as init + component "~__main__" as main + component adapter + component c7nlib + component celparser + component celtypes + component evaluation + component cel.lark + } + init --> celtypes + init --> adapter + init --> celparser + init--> evaluation + + main --> init + main --> celparser + main --> adapter + main --> evaluation + + adapter --> celtypes + + c7nlib --> evaluation + c7nlib --> adapter + c7nlib --> celtypes + c7nlib --> init + + celparser --> cel.lark + celparser --> lark + + evaluation --> lark + evaluation --> celtypes + + package lark { + } + @enduml + +While there is a tangle of dependencies, there are three top-level "entry points" for ``celpy``. + +- The ``__main__`` module is the CLI application. + +- The ``c7nlib`` module exposes CEL functionality in a form usable by Cloud Custodian filter definitions. + This library provides useful components to perform Custodian-related computations. + +- The ``__init__`` module is exposes the most useful parts of ``celpy`` for integration woth another application. + +Compile-Time +------------- + +Here are the essential classes used to compile a CEL expression and prepare it for evaluation. + +.. uml:: + + @startuml + hide empty members + + class Environment { + package: str + annotations: dict[str, Annotation] + compile(text: str) -> lark.Tree + program(expr: lark.Tree, functions: dict) -> Runner + } + + class celparser.CELParser{ + parse(text: str) + } + Environment *-- CELParser + + class lark.Tree {} + CELParser --> lark.Tree : "Creates" + + abstract class Runner { + ast: Tree + evaluate(context: Context) -> Value + } + Environment --> Runner : "Creates" + Runner o-- lark.Tree + Runner o-- "0..m" CELFunction + + class InterpretedRunner + Runner <|-- InterpretedRunner + + class evaluation.Evaluator + InterpretedRunner *-- Evaluator + + class CompiledRunner + Runner <|-- CompiledRunner + + class evaluation.Transpiler + CompiledRunner *-- Transpiler + + class evaluation.Context << (T,orchid) Type>> { + key: str + value: Result | NameContainer + } + Runner o--- "0..m" Context + + class CELFunction <> + + class Annotation << (T,orchid) Type>> + Environment o-- "0..m" Annotation + + class TypeType + Annotation <|-- TypeType + Annotation <|-- CELFunction -- A transient top-most layer is used to create a local variable binding - for the macro evaluations. + @enduml -The AST is created by Lark from the CEL expression. +The fundamental sequence of operations is +1. Create an :py:class:`celpy.Environment` with any needed :py:class:`celpy.Annotation` instances. + For the most part, these are based on the overall application domain. + Any type definitions should be subclasses of :py:class:`celpy.TypeType` or a callable function defined by the :py:class:`celpy.CELFunction` type. + +2. Use the :py:class:`celpy.Environment` to compile the CEL text to create a parse tree. + +3. Use the :py:class:`celpy.Environment` to create a :py:class:`celpy.Runner` instance from the parse tree and any function definitions that override or extend the predefined CEL environment. + +4. Evaluate the :py:class:`celpy.Runner` with a :py:class:`celpy.Context`. + The :py:class:`celpy.Context` provides specific values for variables required for evaluation. + Generally, each variable should have an :py:class:`celpy.Annotation` defined in the :py:class:`celpy.Environment`. + +The :py:class:`celpy.Runner` can be evaluated with any number of distinct :py:class:`celpy.Context` values. +This amortizes the cost of compilation over multiple executions. + +Evaluation-Time +---------------- + +Here's the classes to evaluate a CEL expression. + +.. uml:: + + @startuml + hide empty members + + abstract class Runner { + ast: Tree + evaluate(context: Context) -> Value + } + Environment --- Runner : "Created By <" + Runner o-- "0..m" CELFunction + Runner o-- Context + + class lark.Tree + Tree --* Runner + + class InterpretedRunner <> + Runner <|-- InterpretedRunner + + abstract class lark.Interpreter + + class evaluation.Evaluator { + activation: Activation + functions: dict[str, CELFunction] + evaluate() -> Value + } + lark.Interpreter <|--- Evaluator + InterpretedRunner *-- Evaluator + + class CompiledRunner <> + Runner <|-- CompiledRunner + + InterpretedRunner -[hidden]> CompiledRunner + + class evaluation.Transpiler { + functions: dict[str, CELFunction] + transpile() + evaluate() -> Value + } + CompiledRunner *-- Transpiler + lark.Interpreter <|--- Transpiler + + class evaluation.Activation { + annotations: Annotation + identifiers: dict[str, Result | CELFunction] + } + Runner *-- Activation : "Uses" + Runner --> Activation : "Creates" + Activation --> Activation : "based on" + + class Annotation << (T,orchid) Type>> + Runner *-- "0..m" Annotation + Annotation --o Activation : Initializes + CELFunction --o Activation : Initializes + Context --o Activation : Initializes + + @enduml + +The evalation of the CEL expression is done via a :py:class:`celpy.Runner` object. There are two :py:class:`celpy.Runner` implementations. -- The :py:class:`celpy.InterpretedRunner` walks the AST, creating the final result or exception. +- The :py:class:`celpy.InterpretedRunner` walks the AST, creating the final result :py:class:`celpy.Value` or :py:class:`celpy.CELEvalError` exception. + This uses a :py:class:`celpy.evaluation.Activation` to perform the evaluation. + +- The :py:class:`celpy.CompiledRunner` transpiles the AST into a Python sequence of statements. + The internal :py:func:`compile` creates a code object that can then be evaluated with a given :py:class:`celpy.evaluation.Activation` + The internal :py:func:`exec` functions performs the evaluation. + +The subclasses of :py:class:`celpy.Runner` are **Adapter** classes to provide a tidy interface to the somewhat more complex :py:class:`celpy.Evaluator` or :py:class:`celpy.Transpiler` objects. +In the case of the :py:class:`celpy.InterpretedRunner`, evaluation involves creating an :py:class:`celpy.evaluation.Activation` and visiting the AST. +Whereas, the :py:class:`celpy.CompiledRunner` must first visit the AST to create code. At evaluation time, it create an :py:class:`celpy.evaluation.Activation` and uses :py:func:`exec` to compute the final value. + +The :py:class:`celpy.evaluation.Activation` contains several things: + +- The :py:class:`Annotation` definitions to provide type information for identifiers. -- The :py:class:`celpy.CompiledRunner` transforms the AST to remove empty rules. Then emits - the result as a Python expression. It uses the Python internal :py:func:`compile` and :py:func:`eval` functions - to evaluate the expression. +- The :py:class:`CELFunction` functions that extend or override the built-in functions. + +- The values for identifiers. + +The :py:class:`celpy.evaluation.Activation` is a kind of chainmap for name resolution. +The chain has the following structure: + +- The end of the chain has the built-in defaults. + (This is the bottom-most base definition.) + +- A layer on top of this can offer types and functions which are provided to integrate into the containing app or framework. + +- The next layer is the "current" activation when evaluating a given expression. + For the CLI, this has the command-line variables. + For other integrations, these are the input values. + +- A transient layer on top of this is used to create a local variable binding for the macro evaluations. + These can be nested, and introduce the macro variable as a temporary annotation and value binding. CEL Types -========== +---------- There are ten extension types that wrap Python built-in types to provide the unique CEL semantics. +- :py:class:`celtypes.TypeType` is a supertype for CEL types. + - :py:class:`celtypes.BoolType` wraps ``int`` and creates additional type overload exceptions. - :py:class:`celtypes.BytesType` wraps ``bytes`` it handles conversion from :py:class:`celtypes.StringType`. @@ -73,4 +387,83 @@ There are ten extension types that wrap Python built-in types to provide the uni from ``datetime.timedelta``, ``int``, and ``str`` values. Additionally, a :py:class:`celtypes.NullType` is defined, but does not seem to be needed. It hasn't been deleted, yet. -but should be considered deprecated. +It should be considered deprecated. + +Transpiler Missing Names +==================================================== + +The ``member_dot`` transpilation with a missing name will be found at evaluation time via ``member.get('IDENT')``. This raises No Such Member in Mapping error. + +The ``primary :: ident`` evaluation can result in one of the following conditions: + + - ``ident`` denotes a type definition. The value's type is ``TypeType``. + The value is a type reference ``bool`` becomes ``celpy.celtypes.BoolType``. + + - ``ident`` denotes a built-in function. The value's type is ``CELFunction``. + The value is the Python function reference. + + - ``ident`` denotes an annotation, but the value's type is neither ``TypeType`` nor ``CELFunction``. + + The transpiled value is ``f"activation.{ident}"``, assuming it will be a defined variable. + + If, at ``exec()`` time the name is not in the Activation with a value, a ``NameError`` exception will be raised that becomes a ``CELEvalError`` exception. + + +The Member-Dot Production +========================= + +Consider ``protobuf_message{field: 42}.field``. +This is parsed using the following productions. + +.. code-block:: bnf + + member : member_dot | member_dot_arg | member_item | member_object | primary + member_dot : member "." IDENT + member_object : member "{" [fieldinits] "}" + +The ``member_object`` will be a ``primary`` which can be an ``ident``. +It MUST refer to the Annotation (not the value) because it has ``fieldinits``. +All other choices are (generally) values. +They can be annotations, which means ``bool.type()`` works the same as ``type(bool)``. + +Here's ``primary`` production, which defines the ``ident`` in the ``member`` production. + +.. code-block:: bnf + + primary : dot_ident_arg | dot_ident | ident_arg | ident + | paren_expr | list_lit | map_lit | literal + +The ``ident`` is not **always** transpiled as ``activation.{name}``. +Inside ``member_object``, it's ``activation.resolve_name({name})``. +Outside ``member_object``, it can be ``activation.{name}`` because it's a simple variable. + +It may make sense to rename the :py:meth:`Activation.resolve_name` method to :py:meth:`Activation.get`. + +This, however, overloads the ``get()`` method. +This has type hint consequences. + +.. important:: + + The ``member`` can be any of a variety of objects: + + - ``NameContainer(Dict[str, Referent])`` + + - ``Activation`` + + - ``MapType(Dict[Value, Value])`` + + - ``MessageType(MapType)`` + + All of these classes must define a ``get()`` method. + The nuance is the ``NameContainer`` is also a Python ``dict`` and there's an + overload issue between that ``get()`` and other ``get()`` definitions. + +The Transpilation **currently** leverages a common method named ``get()`` for all of these types. +This is a Pythonic approach, but, the overload for the ``NameContainer`` (a ``Dict`` subclass) isn't quite right: +it doesn't return a ``Referent``, but the value from a ``Referent``. + +A slightly smarter approach is to define a ``get_value(member, 'name')`` function that uses a match/case structure to do the right thing for each type. The problem is, the result is a union of type, value, function, and any of these four containers! + +Another possibility is to leverage the Annotations. +They **can** provide needed type information to discern which method with specific result type. + diff --git a/docs/build/html/api.html b/docs/build/html/api.html index 160caaf..5ede583 100644 --- a/docs/build/html/api.html +++ b/docs/build/html/api.html @@ -5,7 +5,7 @@ - CEL-Py API — CEL in Python documentation + API — CEL in Python documentation @@ -14,8 +14,8 @@ - - + + @@ -33,104 +33,96 @@
-
-

CEL-Py API

+
+

API

Details of the CEL-Python implementation and the API to the various components.

-

Pure Python implementation of CEL.

-

Visible interface to CEL. This exposes the Environment, -the Evaluator run-time, and the celtypes module -with Python types wrapped to be CEL compatible.

-
-

Example

-

Here’s an example with some details:

-
>>> import celpy
-
-# A list of type names and class bindings used to create an environment.
->>> types = []
->>> env = celpy.Environment(types)
-
-# Parse the code to create the CEL AST.
->>> ast = env.compile("355. / 113.")
-
-# Use the AST and any overriding functions to create an executable program.
->>> functions = {}
->>> prgm = env.program(ast, functions)
-
-# Variable bindings.
->>> activation = {}
-
-# Final evaluation.
->>> try:
-...    result = prgm.evaluate(activation)
-...    error = None
-... except CELEvalError as ex:
-...    result = None
-...    error = ex.args[0]
-
->>> result
-DoubleType(3.14159...)
-
-
-
-
-

Another Example

-

See https://github.com/google/cel-go/blob/master/examples/simple_test.go

-

The model Go we’re sticking close to:

-
d := cel.Declarations(decls.NewVar("name", decls.String))
-env, err := cel.NewEnv(d)
-if err != nil {
-    log.Fatalf("environment creation error: %v\n", err)
-}
-ast, iss := env.Compile(`"Hello world! I'm " + name + "."`)
-// Check iss for compilation errors.
-if iss.Err() != nil {
-    log.Fatalln(iss.Err())
-}
-prg, err := env.Program(ast)
-if err != nil {
-    log.Fatalln(err)
-}
-out, _, err := prg.Eval(map[string]interface{}{
-    "name": "CEL",
-})
-if err != nil {
-    log.Fatalln(err)
-}
-fmt.Println(out)
-// Output:Hello world! I'm CEL.
-
-
-

Here’s the Pythonic approach, using concept patterned after the Go implementation:

-
>>> from celpy import *
->>> decls = {"name": celtypes.StringType}
->>> env = Environment(annotations=decls)
->>> ast = env.compile('"Hello world! I\'m " + name + "."')
->>> out = env.program(ast).evaluate({"name": "CEL"})
->>> print(out)
-Hello world! I'm CEL.
-
-
-
+

The following modules are described here:

+ +
+

celpy

+

The pure Python implementation of the Common Expression Language, CEL.

+

This module defines an interface to CEL for integration into other Python applications. +This exposes the Environment used to compile the source module, +the Runner used to evaluate the compiled code, +and the celpy.celtypes module with Python types wrapped to be CEL compatible.

+

The way these classes are used is as follows:

+

+@startuml
+start
+:Gather (or define) annotations;
+:Create ""Environment"";
+:Compile CEL;
+:Create ""Runner"" with any extension functions;
+:Evaluate the ""Runner"" with a ""Context"";
+stop
+@enduml +

+

The explicit decomposition into steps permits +two extensions:

+
    +
  1. Transforming the AST to introduce any optimizations.

  2. +
  3. Saving the Runner instance to reuse an expression with new inputs.

  4. +
class celpy.__init__.Runner(environment: Environment, ast: Tree, functions: Dict[str, Callable[[...], BoolType | BytesType | DoubleType | DurationType | IntType | ListType | MapType | None | StringType | TimestampType | UintType | CELEvalError | Type[BoolType] | Type[BytesType] | Type[DoubleType] | Type[DurationType] | Type[IntType] | Type[ListType] | Type[MapType] | Callable[[...], None] | Type[StringType] | Type[TimestampType] | Type[TypeType] | Type[UintType] | Type[PackageType] | Type[MessageType]]] | None = None)[source]
-

Abstract runner.

-

Given an AST, this can evaluate the AST in the context of a specific activation -with any override function definitions.

+

Abstract runner for a compiled CEL program.

+

The Environment creates a Runner to permit +saving a ready-tp-evaluate, compiled CEL expression. +A Runner will evaluate the AST in the context of a specific activation +with the provided variable values.

+

A Runner class provides +the tree_node_class attribute to define the type for tree nodes. +This class information is used by the Environment to tailor the Lark instance created. +The class named by the tree_node_class can include specialized AST features +needed by a Runner instance.

+
+
+tree_node_class
+

alias of Tree

+
+
__init__(environment: Environment, ast: Tree, functions: Dict[str, Callable[[...], BoolType | BytesType | DoubleType | DurationType | IntType | ListType | MapType | None | StringType | TimestampType | UintType | CELEvalError | Type[BoolType] | Type[BytesType] | Type[DoubleType] | Type[DurationType] | Type[IntType] | Type[ListType] | Type[MapType] | Callable[[...], None] | Type[StringType] | Type[TimestampType] | Type[TypeType] | Type[UintType] | Type[PackageType] | Type[MessageType]]] | None = None) None[source]
-
+

Initialize this Runner with a given AST. +The Runner will have annotations take from the Environment, +plus any unique functions defined here.

+
+ +
+
+__repr__() str[source]
+

Return repr(self).

+
-new_activation(context: Mapping[str, BoolType | BytesType | DoubleType | DurationType | IntType | ListType | MapType | None | StringType | TimestampType | UintType | CELEvalError | Type[BoolType] | Type[BytesType] | Type[DoubleType] | Type[DurationType] | Type[IntType] | Type[ListType] | Type[MapType] | Callable[[...], None] | Type[StringType] | Type[TimestampType] | Type[TypeType] | Type[UintType] | Type[PackageType] | Type[MessageType] | NameContainer]) Activation[source]
-

Builds the working activation from the environmental defaults.

+new_activation() Activation[source] +

Builds a new, working Activation using the Environment as defaults. +A Context will later be layered onto this for evaluation.

-evaluate(activation: Mapping[str, BoolType | BytesType | DoubleType | DurationType | IntType | ListType | MapType | None | StringType | TimestampType | UintType | CELEvalError | Type[BoolType] | Type[BytesType] | Type[DoubleType] | Type[DurationType] | Type[IntType] | Type[ListType] | Type[MapType] | Callable[[...], None] | Type[StringType] | Type[TimestampType] | Type[TypeType] | Type[UintType] | Type[PackageType] | Type[MessageType] | NameContainer]) BoolType | BytesType | DoubleType | DurationType | IntType | ListType | MapType | None | StringType | TimestampType | UintType[source]
+abstractmethod evaluate(activation: Mapping[str, BoolType | BytesType | DoubleType | DurationType | IntType | ListType | MapType | None | StringType | TimestampType | UintType | CELEvalError | Type[BoolType] | Type[BytesType] | Type[DoubleType] | Type[DurationType] | Type[IntType] | Type[ListType] | Type[MapType] | Callable[[...], None] | Type[StringType] | Type[TimestampType] | Type[TypeType] | Type[UintType] | Type[PackageType] | Type[MessageType] | NameContainer | CELFunction]) BoolType | BytesType | DoubleType | DurationType | IntType | ListType | MapType | None | StringType | TimestampType | UintType[source] +

Given variable definitions in the celpy.evaluation.Context, evaluate the given AST and return the resulting value.

+

Generally, this should raise an CELEvalError for most kinds of ordinary problems. +It may raise an CELUnsupportedError for future features that aren’t fully implemented. +Any Python exception reflects a serious problem.

+
+ +
+
+__abstractmethods__ = frozenset({'evaluate'})
@@ -138,33 +130,53 @@

Another Example
class celpy.__init__.InterpretedRunner(environment: Environment, ast: Tree, functions: Dict[str, Callable[[...], BoolType | BytesType | DoubleType | DurationType | IntType | ListType | MapType | None | StringType | TimestampType | UintType | CELEvalError | Type[BoolType] | Type[BytesType] | Type[DoubleType] | Type[DurationType] | Type[IntType] | Type[ListType] | Type[MapType] | Callable[[...], None] | Type[StringType] | Type[TimestampType] | Type[TypeType] | Type[UintType] | Type[PackageType] | Type[MessageType]]] | None = None)[source]
-

Pure AST expression evaluator. Uses evaluation.Evaluator class.

-

Given an AST, this evauates the AST in the context of a specific activation.

-

The returned value will be a celtypes type.

-

Generally, this should raise an CELEvalError for most kinds of ordinary problems. -It may raise an CELUnsupportedError for future features.

+

An Adapter for the celpy.evaluation.Evaluator class.

-evaluate(context: Mapping[str, BoolType | BytesType | DoubleType | DurationType | IntType | ListType | MapType | None | StringType | TimestampType | UintType | CELEvalError | Type[BoolType] | Type[BytesType] | Type[DoubleType] | Type[DurationType] | Type[IntType] | Type[ListType] | Type[MapType] | Callable[[...], None] | Type[StringType] | Type[TimestampType] | Type[TypeType] | Type[UintType] | Type[PackageType] | Type[MessageType] | NameContainer]) BoolType | BytesType | DoubleType | DurationType | IntType | ListType | MapType | None | StringType | TimestampType | UintType[source]
+evaluate(context: Mapping[str, BoolType | BytesType | DoubleType | DurationType | IntType | ListType | MapType | None | StringType | TimestampType | UintType | CELEvalError | Type[BoolType] | Type[BytesType] | Type[DoubleType] | Type[DurationType] | Type[IntType] | Type[ListType] | Type[MapType] | Callable[[...], None] | Type[StringType] | Type[TimestampType] | Type[TypeType] | Type[UintType] | Type[PackageType] | Type[MessageType] | NameContainer | CELFunction]) BoolType | BytesType | DoubleType | DurationType | IntType | ListType | MapType | None | StringType | TimestampType | UintType[source] +

Given variable definitions in the celpy.evaluation.Context, evaluate the given AST and return the resulting value.

+

Generally, this should raise an CELEvalError for most kinds of ordinary problems. +It may raise an CELUnsupportedError for future features that aren’t fully implemented. +Any Python exception reflects a serious problem.

+
+ +
+
+__abstractmethods__ = frozenset({})
-class celpy.__init__.CompiledRunner(environment: Environment, ast: Tree, functions: Dict[str, Callable[[...], BoolType | BytesType | DoubleType | DurationType | IntType | ListType | MapType | None | StringType | TimestampType | UintType | CELEvalError | Type[BoolType] | Type[BytesType] | Type[DoubleType] | Type[DurationType] | Type[IntType] | Type[ListType] | Type[MapType] | Callable[[...], None] | Type[StringType] | Type[TimestampType] | Type[TypeType] | Type[UintType] | Type[PackageType] | Type[MessageType]]] | None = None)[source]
-

Python compiled expression evaluator. Uses Python byte code and eval().

-

Given an AST, this evaluates the AST in the context of a specific activation.

-

Transform the AST into Python, uses compile() to create a code object. -Uses eval() to evaluate.

+class celpy.__init__.CompiledRunner(environment: Environment, ast: TranspilerTree, functions: Dict[str, Callable[[...], BoolType | BytesType | DoubleType | DurationType | IntType | ListType | MapType | None | StringType | TimestampType | UintType | CELEvalError | Type[BoolType] | Type[BytesType] | Type[DoubleType] | Type[DurationType] | Type[IntType] | Type[ListType] | Type[MapType] | Callable[[...], None] | Type[StringType] | Type[TimestampType] | Type[TypeType] | Type[UintType] | Type[PackageType] | Type[MessageType]]] | None = None)[source] +

An Adapter for the celpy.evaluation.Transpiler class.

+

A celpy.evaluation.Transpiler instance transforms the AST into Python. +It uses compile() to create a code object. +The final evaluate() method uses exec() to evaluate the code object.

+

Note, this requires the celpy.evaluation.TranspilerTree classes +instead of the default lark.Tree class.

+
+
+tree_node_class
+

alias of TranspilerTree

+
+
-__init__(environment: Environment, ast: Tree, functions: Dict[str, Callable[[...], BoolType | BytesType | DoubleType | DurationType | IntType | ListType | MapType | None | StringType | TimestampType | UintType | CELEvalError | Type[BoolType] | Type[BytesType] | Type[DoubleType] | Type[DurationType] | Type[IntType] | Type[ListType] | Type[MapType] | Callable[[...], None] | Type[StringType] | Type[TimestampType] | Type[TypeType] | Type[UintType] | Type[PackageType] | Type[MessageType]]] | None = None) None[source]
-
+__init__(environment: Environment, ast: TranspilerTree, functions: Dict[str, Callable[[...], BoolType | BytesType | DoubleType | DurationType | IntType | ListType | MapType | None | StringType | TimestampType | UintType | CELEvalError | Type[BoolType] | Type[BytesType] | Type[DoubleType] | Type[DurationType] | Type[IntType] | Type[ListType] | Type[MapType] | Callable[[...], None] | Type[StringType] | Type[TimestampType] | Type[TypeType] | Type[UintType] | Type[PackageType] | Type[MessageType]]] | None = None) None[source] +

Transpile to Python, and use compile() to create a code object.

+
-evaluate(activation: Mapping[str, BoolType | BytesType | DoubleType | DurationType | IntType | ListType | MapType | None | StringType | TimestampType | UintType | CELEvalError | Type[BoolType] | Type[BytesType] | Type[DoubleType] | Type[DurationType] | Type[IntType] | Type[ListType] | Type[MapType] | Callable[[...], None] | Type[StringType] | Type[TimestampType] | Type[TypeType] | Type[UintType] | Type[PackageType] | Type[MessageType] | NameContainer]) BoolType | BytesType | DoubleType | DurationType | IntType | ListType | MapType | None | StringType | TimestampType | UintType[source]
+evaluate(context: Mapping[str, BoolType | BytesType | DoubleType | DurationType | IntType | ListType | MapType | None | StringType | TimestampType | UintType | CELEvalError | Type[BoolType] | Type[BytesType] | Type[DoubleType] | Type[DurationType] | Type[IntType] | Type[ListType] | Type[MapType] | Callable[[...], None] | Type[StringType] | Type[TimestampType] | Type[TypeType] | Type[UintType] | Type[PackageType] | Type[MessageType] | NameContainer | CELFunction]) BoolType | BytesType | DoubleType | DurationType | IntType | ListType | MapType | None | StringType | TimestampType | UintType[source] +

Use exec() to execute the code object.

+
+ +
+
+__abstractmethods__ = frozenset({})
@@ -172,7 +184,8 @@

Another Example
class celpy.__init__.Int32Value(value: Any = 0)[source]
-
+

A wrapper for int32 values.

+
static __new__(cls: Type[Int32Value], value: Any = 0) Int32Value[source]

TODO: Check range. This seems to matter for protobuf.

@@ -182,16 +195,15 @@

Another Example
-class celpy.__init__.Environment(package: str | None = None, annotations: Dict[str, Type[BoolType] | Type[BytesType] | Type[DoubleType] | Type[DurationType] | Type[IntType] | Type[ListType] | Type[MapType] | Callable[[...], None] | Type[StringType] | Type[TimestampType] | Type[TypeType] | Type[UintType] | Type[PackageType] | Type[MessageType] | Callable[[...], BoolType | BytesType | DoubleType | DurationType | IntType | ListType | MapType | None | StringType | TimestampType | UintType] | Type[FunctionType]] | None = None, runner_class: Type[Runner] | None = None)[source]
-

Compiles CEL text to create an Expression object.

-

From the Go implementation, there are things to work with the type annotations:

-
    -
  • type adapters registry make other native types available for CEL.

  • -
  • type providers registry make ProtoBuf types available for CEL.

  • -
+class celpy.__init__.Environment(package: str | None = None, annotations: Dict[str, TypeType | Callable[[...], BoolType | BytesType | DoubleType | DurationType | IntType | ListType | MapType | None | StringType | TimestampType | UintType] | Type[FunctionType]] | None = None, runner_class: Type[Runner] | None = None)[source] +

Contains the current evaluation context. +The Environment.compile() method +compiles CEL text to create an AST.

+

The Environment.program() method +packages the AST into a program ready for evaluation.

-__init__(package: str | None = None, annotations: Dict[str, Type[BoolType] | Type[BytesType] | Type[DoubleType] | Type[DurationType] | Type[IntType] | Type[ListType] | Type[MapType] | Callable[[...], None] | Type[StringType] | Type[TimestampType] | Type[TypeType] | Type[UintType] | Type[PackageType] | Type[MessageType] | Callable[[...], BoolType | BytesType | DoubleType | DurationType | IntType | ListType | MapType | None | StringType | TimestampType | UintType] | Type[FunctionType]] | None = None, runner_class: Type[Runner] | None = None) None[source]
+__init__(package: str | None = None, annotations: Dict[str, TypeType | Callable[[...], BoolType | BytesType | DoubleType | DurationType | IntType | ListType | MapType | None | StringType | TimestampType | UintType] | Type[FunctionType]] | None = None, runner_class: Type[Runner] | None = None) None[source]

Create a new environment.

This also increases the default recursion limit to handle the defined minimums for CEL.

@@ -211,6 +223,12 @@

Another Example +
+__repr__() str[source]
+

Return repr(self).

+

+
compile(text: str) Tree[source]
@@ -220,73 +238,31 @@

Another Example
program(expr: Tree, functions: Dict[str, Callable[[...], BoolType | BytesType | DoubleType | DurationType | IntType | ListType | MapType | None | StringType | TimestampType | UintType | CELEvalError | Type[BoolType] | Type[BytesType] | Type[DoubleType] | Type[DurationType] | Type[IntType] | Type[ListType] | Type[MapType] | Callable[[...], None] | Type[StringType] | Type[TimestampType] | Type[TypeType] | Type[UintType] | Type[PackageType] | Type[MessageType]]] | None = None) Runner[source]
-

Transforms the AST into an executable runner.

-

- -
-
-activation() Activation[source]
-

Returns a base activation

+

Transforms the AST into an executable Runner object.

+
+
Parameters:
+
    +
  • expr – The parse tree from compile().

  • +
  • functions – Any additional functions to be used by this CEL expression.

  • +
+
+
Returns:
+

A Runner instance that can be evaluated with a Context that provides values.

+
+
+

__main__

-

Pure Python implementation of CEL.

-

This provides a few jq-like, bc-like, and shell expr-like features.

-
    -
  • jq uses . to refer the current document. By setting a package -name of "jq" and placing the JSON object in the package, we achieve -similar syntax.

  • -
  • bc offers complex function definitions and other programming support. -CEL can only evaluate a few bc-like expressions.

  • -
  • This does everything expr does, but the syntax is slightly different. -The output of comparisons – by default – is boolean, where expr is an integer 1 or 0. -Use -f 'd' to see decimal output instead of Boolean text values.

  • -
  • This does some of what test does, without a lot of the sophisticated -file system data gathering. -Use -b to set the exit status code from a Boolean result.

  • -
-

TODO: This can also have a REPL, as well as process CSV files.

-
-

SYNOPSIS

-
python -m celpy [--arg name:type=value ...] [--null-input] expr
-
-
-

Options:

-
-
–arg:
-

Provides argument names, types and optional values. -If the value is not provided, the name is expected to be an environment -variable, and the value of the environment variable is converted and used.

-
-
–null-input:
-

Normally, JSON documents are read from stdin in ndjson format. If no JSON documents are -provided, the --null-input option skips trying to read from stdin.

-
-
expr:
-

A CEL expression to evaluate.

-
-
-

JSON documents are read from stdin in NDJSON format (http://jsonlines.org/, http://ndjson.org/). -For each JSON document, the expression is evaluated with the document in a default -package. This allows .name to pick items from the document.

-

By default, the output is JSON serialized. This means strings will be JSON-ified and have quotes.

-

If a --format option is provided, this is applied to the resulting object; this can be -used to strip quotes, or limit precision on double objects, or convert numbers to hexadecimal.

-
-
-

Arguments, Types, and Namespaces

-

CEL objects rely on the celtypes definitions.

-

Because of the close association between CEL and protobuf, some well-known protobuf types -are also supported.

-

Further, type providers can be bound to CEL. This means an extended CEL -may have additional types beyond those defined by the Activation class.

-
+

The CLI interface to celpy.

+

This parses the command-line options. +It also offers an interactive REPL.

-celpy.__main__.arg_type_value(text: str) Tuple[str, Type[BoolType] | Type[BytesType] | Type[DoubleType] | Type[DurationType] | Type[IntType] | Type[ListType] | Type[MapType] | Callable[[...], None] | Type[StringType] | Type[TimestampType] | Type[TypeType] | Type[UintType] | Type[PackageType] | Type[MessageType] | Callable[[...], BoolType | BytesType | DoubleType | DurationType | IntType | ListType | MapType | None | StringType | TimestampType | UintType] | Type[FunctionType], BoolType | BytesType | DoubleType | DurationType | IntType | ListType | MapType | None | StringType | TimestampType | UintType][source]
+celpy.__main__.arg_type_value(text: str) Tuple[str, TypeType | Callable[[...], BoolType | BytesType | DoubleType | DurationType | IntType | ListType | MapType | None | StringType | TimestampType | UintType] | Type[FunctionType], BoolType | BytesType | DoubleType | DurationType | IntType | ListType | MapType | None | StringType | TimestampType | UintType][source]

Decompose -a name:type=value argument into a useful triple.

Also accept -a name:type. This will find name in the environment and convert to the requested type.

@@ -321,6 +297,12 @@

Arguments, Types, and Namespaces +
+celpy.__main__.stat(path: Path | str) MapType | None[source]
+

This function is added to the CLI to permit file-system interrogation.

+

+
class celpy.__main__.CEL_REPL(completekey='tab', stdin=None, stdout=None)[source]
@@ -381,6 +363,12 @@

Arguments, Types, and Namespaces +
+do_EOF(args: str) bool
+

Quits from the REPL.

+

+
default(args: str) None[source]
@@ -416,158 +404,1439 @@

Arguments, Types, and Namespaces -

celtypes

-

CEL Types: wrappers on Python types to provide CEL semantics.

-

This can be used by a Python module to work with CEL-friendly values and CEL results.

-

Examples of distinctions between CEL and Python:

-
    -
  • Unlike Python bool, CEL BoolType won’t do some math.

  • -
  • CEL has int64 and uint64 subclasses of integer. These have specific ranges and -raise ValueError errors on overflow.

  • -
-

CEL types will raise ValueError for out-of-range values and TypeError -for operations they refuse. -The evaluation module can capture these exceptions and turn them into result values. -This can permit the logic operators to quietly silence them via “short-circuiting”.

-

In the normal course of events, CEL’s evaluator may attempt operations between a -CEL exception result and an instance of one of CEL types. -We rely on this leading to an ordinary Python TypeError to be raised to propogate -the error. Or. A logic operator may discard the error object.

-

The evaluation module extends these types with it’s own CELEvalError exception. -We try to keep that as a separate concern from the core operator implementations here. -We leverage Python features, which means raising exceptions when there is a problem.

-
-

Types

-

See https://github.com/google/cel-go/tree/master/common/types

-

These are the Go type definitions that are used by CEL:

-
    -
  • BoolType

  • -
  • BytesType

  • -
  • DoubleType

  • -
  • DurationType

  • -
  • IntType

  • -
  • ListType

  • -
  • MapType

  • -
  • NullType

  • -
  • StringType

  • -
  • TimestampType

  • -
  • TypeType

  • -
  • UintType

  • -
-

The above types are handled directly byt CEL syntax. -e.g., 42 vs. 42u vs. "42" vs. b"42" vs. 42..

-

We provide matching Python class names for each of these types. The Python type names -are subclasses of Python native types, allowing a client to transparently work with -CEL results. A Python host should be able to provide values to CEL that will be tolerated.

-

A type hint of Value unifies these into a common hint.

-

The CEL Go implementation also supports protobuf types:

-
    -
  • dpb.Duration

  • -
  • tpb.Timestamp

  • -
  • structpb.ListValue

  • -
  • structpb.NullValue

  • -
  • structpb.Struct

  • -
  • structpb.Value

  • -
  • wrapperspb.BoolValue

  • -
  • wrapperspb.BytesValue

  • -
  • wrapperspb.DoubleValue

  • -
  • wrapperspb.FloatValue

  • -
  • wrapperspb.Int32Value

  • -
  • wrapperspb.Int64Value

  • -
  • wrapperspb.StringValue

  • -
  • wrapperspb.UInt32Value

  • -
  • wrapperspb.UInt64Value

  • -
-

These types involve expressions like the following:

-
google.protobuf.UInt32Value{value: 123u}
+
+

adapter

+

Converts some Python-native types into CEL structures.

+

Currently, atomic Python objects have direct use of types in celpy.celtypes.

+

Non-atomic Python objects are characterized by JSON and Protobuf messages. +This module has functions to convert JSON objects to CEL.

+

A proper protobuf decoder is TBD.

+

A more sophisticated type injection capability may be needed to permit +additional types or extensions to celpy.celtypes.

+
+
+class celpy.adapter.CELJSONEncoder(*, skipkeys=False, ensure_ascii=True, check_circular=True, allow_nan=True, sort_keys=False, indent=None, separators=None, default=None)[source]
+

An Encoder to export CEL objects as JSON text.

+

This is not a reversible transformation. Some things are coerced to strings +without any more detailed type marker. +Specifically timestamps, durations, and bytes.

+
+
+static to_python(cel_object: BoolType | BytesType | DoubleType | DurationType | IntType | ListType | MapType | None | StringType | TimestampType | UintType) BoolType | BytesType | DoubleType | DurationType | IntType | ListType | MapType | None | StringType | TimestampType | UintType | List[Any] | Dict[Any, Any] | bool[source]
+

Recursive walk through the CEL object, replacing BoolType with native bool instances. +This lets the json module correctly represent the obects +with JSON true and false.

+

This will also replace ListType and MapType with native list and dict. +All other CEL objects will be left intact. This creates an intermediate hybrid +beast that’s not quite a celtypes.Value because a few things have been replaced.

+
+ +
+
+encode(cel_object: BoolType | BytesType | DoubleType | DurationType | IntType | ListType | MapType | None | StringType | TimestampType | UintType) str[source]
+

Override built-in encode to create proper Python bool objects.

+
+ +
+
+default(cel_object: BoolType | BytesType | DoubleType | DurationType | IntType | ListType | MapType | None | StringType | TimestampType | UintType) Dict[str, Any] | List[Any] | bool | float | int | str | None[source]
+

Implement this method in a subclass such that it returns +a serializable object for o, or calls the base implementation +(to raise a TypeError).

+

For example, to support arbitrary iterators, you could +implement default like this:

+
def default(self, o):
+    try:
+        iterable = iter(o)
+    except TypeError:
+        pass
+    else:
+        return list(iterable)
+    # Let the base class default method raise the TypeError
+    return super().default(o)
 
-

In this case, the well-known protobuf name is directly visible as CEL syntax. -There’s a google package with the needed definitions.

-
-
-

Type Provider

-

A type provider can be bound to the environment, this will support additional types. -This appears to be a factory to map names of types to type classes.

-

Run-time type binding is shown by a CEL expression like the following:

-
TestAllTypes{single_uint32_wrapper: 432u}
+

+ + + +
+
+class celpy.adapter.CELJSONDecoder(*, object_hook=None, parse_float=None, parse_int=None, parse_constant=None, strict=True, object_pairs_hook=None)[source]
+

An Encoder to import CEL objects from JSON to the extent possible.

+

This does not handle non-JSON types in any form. Coercion from string +to TimestampType or DurationType or BytesType is handled by celtype +constructors.

+
+
+decode(source: str, _w: Any = None) Any[source]
+

Return the Python representation of s (a str instance +containing a JSON document).

+
+ +
+ +
+
+celpy.adapter.json_to_cel(document: Dict[str, Any] | List[Any] | bool | float | int | str | None) BoolType | BytesType | DoubleType | DurationType | IntType | ListType | MapType | None | StringType | TimestampType | UintType[source]
+

Convert parsed JSON object from Python to CEL to the extent possible.

+

It’s difficult to distinguish strings which should be timestamps or durations.

+
>>> from pprint import pprint
+>>> from celpy.adapter import json_to_cel
+>>> doc = json.loads('["str", 42, 3.14, null, true, {"hello": "world"}]')
+>>> cel = json_to_cel(doc)
+>>> pprint(cel)
+ListType([StringType('str'), IntType(42), DoubleType(3.14), None, BoolType(True), MapType({StringType('hello'): StringType('world')})])
 
-

The TestAllTypes is a protobuf type added to the CEL run-time. The syntax -is defined by this syntax rule:

-
member_object  : member "{" [fieldinits] "}"
+
+ +
+
+

c7nlib

+

Functions for C7N features when evaluating CEL expressions.

+

These functions provide a mapping between C7N features and CEL.

+

These functions are exposed by the global FUNCTIONS dictionary that is provided +to the CEL evaluation run-time to provide necessary C7N features.

+

The functions rely on implementation details in the CELFilter class.

+
+

The API

+

A C7N implementation can use CEL expressions and the c7nlib module as follows:

+
class CELFilter(c7n.filters.core.Filter):
+    decls = {
+        "resource": celpy.celtypes.MapType,
+        "now": celpy.celtypes.TimestampType,
+        "event": celpy.celtypes.MapType,
+    }
+    decls.update(celpy.c7nlib.DECLARATIONS)
+
+    def __init__(self, expr: str) -> None:
+        self.expr = expr
+
+    def validate(self) -> None:
+        cel_env = celpy.Environment(
+            annotations=self.decls,
+            runner_class=c7nlib.C7N_Interpreted_Runner)
+        cel_ast = cel_env.compile(self.expr)
+        self.pgm = cel_env.program(cel_ast, functions=celpy.c7nlib.FUNCTIONS)
+
+    def process(self,
+        resources: Iterable[celpy.celtypes.MapType]) -> Iterator[celpy.celtypes.MapType]:
+        now = datetime.datetime.utcnow()
+        for resource in resources:
+            with C7NContext(filter=the_filter):
+                cel_activation = {
+                    "resource": celpy.json_to_cel(resource),
+                    "now": celpy.celtypes.TimestampType(now),
+                }
+                if self.pgm.evaluate(cel_activation):
+                    yield resource
 
-

The member is part of a type provider library, -either a standard protobuf definition or an extension. The field inits build -values for the protobuf object.

-

See https://github.com/google/cel-go/blob/master/test/proto3pb/test_all_types.proto -for the TestAllTypes protobuf definition that is registered as a type provider.

-

This expression will describes a Protobuf uint32 object.

+

The celpy.c7nlib library of functions is bound into the CEL celpy.__init__.Runner object that’s built from the AST.

+

Several variables will be required in the celpy.evaluation.Activation for use by most CEL expressions +that implement C7N filters:

+
+
resource:
+

A JSON document describing a cloud resource.

+
+
now:
+

The current timestamp.

+
+
event:
+

May be needed; it should be a JSON document describing an AWS CloudWatch event.

+
+
-
-

Type Adapter

-

So far, it appears that a type adapter wraps existing Go or C++ types -with CEL-required methods. This seems like it does not need to be implemented -in Python.

+
+

The type: value Features

+

The core value features of C7N require a number of CEL extensions.

+
    +
  • glob(string, pattern)() uses Python fnmatch rules. This implements op: glob.

  • +
  • difference(list, list)() creates intermediate sets and computes the difference +as a boolean value. Any difference is True. This implements op: difference.

  • +
  • intersect(list, list)() creats intermediate sets and computes the intersection +as a boolean value. Any interection is True. This implements op: intersect.

  • +
  • normalize(string)() supports normalized comparison between strings. +In this case, it means lower cased and trimmed. This implements value_type: normalize.

  • +
  • net.cidr_contains() checks to see if a given CIDR block contains a specific +address. See https://www.openpolicyagent.org/docs/latest/policy-reference/#net.

  • +
  • net.cidr_size() extracts the prefix length of a parsed CIDR block.

  • +
  • version() uses disutils.version.LooseVersion to compare version strings.

  • +
  • resource_count() function. This is TBD.

  • +
-
-

Numeric Details

-

Integer division truncates toward zero.

-

The Go definition of modulus:

-
// Mod returns the floating-point remainder of x/y.
-// The magnitude of the result is less than y and its
-// sign agrees with that of x.
+
+

The type: value_from features

+

This relies on value_from() and jmes_path_map() functions

+

In context, it looks like this:

+
value_from("s3://c7n-resources/exemptions.json", "json")
+.jmes_path_map('exemptions.ec2.rehydration.["IamInstanceProfile.Arn"][].*[].*[]')
+.contains(resource["IamInstanceProfile"]["Arn"])
 
-

https://golang.org/ref/spec#Arithmetic_operators

-

“Go has the nice property that -a/b == -(a/b).”

-
 x     y     x / y     x % y
- 5     3       1         2
--5     3      -1        -2
- 5    -3      -1         2
--5    -3       1        -2
+

The value_from() function reads values from a given URI.

+
    +
  • A full URI for an S3 bucket.

  • +
  • A full URI for a server that supports HTTPS GET requests.

  • +
+

If a format is given, this is used, otherwise it’s based on the +suffix of the path.

+

The jmes_path_map() function compiles and applies a JMESPath +expression against each item in the collection to create a +new collection. To an extent, this repeats functionality +from the map() macro.

+
+
+

Additional Functions

+

A number of C7N subclasses of Filter provide additional features. There are +at least 70-odd functions that are expressed or implied by these filters.

+

Because the CEL expressions are always part of a CELFilter, all of these +additional C7N features need to be transformed into “mixins” that are implemented +in two places. The function is part of the legacy subclass of Filter, +and the function is also part of CELFilter.

+
class InstanceImageMixin:
+    # from :py:class:`InstanceImageBase` refactoring
+    def get_instance_image(self):
+        pass
+
+class RelatedResourceMixin:
+    # from :py:class:`RelatedResourceFilter` mixin
+    def get_related_ids(self):
+        pass
+
+    def get_related(self):
+        pass
+
+class CredentialReportMixin:
+    # from :py:class:`c7n.resources.iam.CredentialReport` filter.
+    def get_credential_report(self):
+        pass
+
+class ResourceKmsKeyAliasMixin:
+    # from :py:class:`c7n.resources.kms.ResourceKmsKeyAlias`
+    def get_matching_aliases(self, resource):
+        pass
+
+class CrossAccountAccessMixin:
+    # from :py:class:`c7n.filters.iamaccessfilter.CrossAccountAccessFilter`
+    def get_accounts(self, resource):
+        pass
+    def get_vpcs(self, resource):
+        pass
+    def get_vpces(self, resource):
+        pass
+    def get_orgids(self, resource):
+        pass
+    # from :py:class:`c7n.resources.secretsmanager.CrossAccountAccessFilter`
+    def get_resource_policy(self, resource):
+        pass
+
+class SNSCrossAccountMixin:
+    # from :py:class:`c7n.resources.sns.SNSCrossAccount`
+    def get_endpoints(self, resource):
+        pass
+    def get_protocols(self, resource):
+        pass
+
+class ImagesUnusedMixin:
+    # from :py:class:`c7n.resources.ami.ImageUnusedFilter`
+    def _pull_ec2_images(self, resource):
+        pass
+    def _pull_asg_images(self, resource):
+        pass
+
+class SnapshotUnusedMixin:
+    # from :py:class:`c7n.resources.ebs.SnapshotUnusedFilter`
+    def _pull_asg_snapshots(self, resource):
+        pass
+    def _pull_ami_snapshots(self, resource):
+        pass
+
+class IamRoleUsageMixin:
+    # from :py:class:`c7n.resources.iam.IamRoleUsage`
+    def service_role_usage(self, resource):
+        pass
+    def instance_profile_usage(self, resource):
+        pass
+
+class SGUsageMixin:
+    # from :py:class:`c7n.resources.vpc.SGUsage`
+    def scan_groups(self, resource):
+        pass
+
+class IsShieldProtectedMixin:
+    # from :py:mod:`c7n.resources.shield`
+    def get_type_protections(self, resource):
+        pass
+
+class ShieldEnabledMixin:
+    # from :py:class:`c7n.resources.account.ShieldEnabled`
+    def account_shield_subscriptions(self, resource):
+        pass
+
+class CELFilter(
+    InstanceImageMixin, RelatedResourceMixin, CredentialReportMixin,
+    ResourceKmsKeyAliasMixin, CrossAccountAccessMixin, SNSCrossAccountMixin,
+    ImagesUnusedMixin, SnapshotUnusedMixin, IamRoleUsageMixin, SGUsageMixin,
+    Filter,
+):
+    '''Container for functions used by c7nlib to expose data to CEL'''
+    def __init__(self, data, manager) -> None:
+        super().__init__(data, manager)
+        assert data["type"].lower() == "cel"
+        self.expr = data["expr"]
+        self.parser = c7n.filters.offhours.ScheduleParser()
+
+    def validate(self):
+        pass  # See above example
+
+    def process(self, resources):
+        pass  # See above example
 
-

Python definition:

-
The modulo operator always yields a result
-with the same sign as its second operand (or zero);
-the absolute value of the result is strictly smaller than
-the absolute value of the second operand.
+

This is not the complete list. See the tests/test_c7nlib.py for the celfilter_instance +fixture which contains all of the functions required.

+
+
+

C7N Context Object

+

A number of the functions require access to C7N features that are not simply part +of the resource being filtered. There are two alternative ways to handle this dependency:

+
    +
  • A global C7N context object that has the current CELFilter providing +access to C7N internals.

  • +
  • A C7N argument to the functions that need C7N access. +This would be provided in the activation context for CEL.

  • +
+

To keep the library functions looking simple, the module global C7N is used. +This avoids introducing a non-CEL parameter to the celpy.c7nlib functions.

+

The C7N context object contains the following attributes:

+
+
filter:
+

The original C7N Filter object. This provides access to the +resource manager. It can be used to manage supplemental +queries using C7N caches and other resource management.

+
+
+

This is set by the C7NContext prior to CEL evaluation.

+
+
+

Name Resolution

+

Note that names are not resolved via a lookup in the program object, +an instance of the celpy.Runner class. To keep these functions +simple, the runner is not part of the run-time, and name resolution +will appear to be “hard-wrired” among these functions.

+

This is rarely an issue, since most of these functions are independent. +The value_from() function relies on text_from() and parse_text(). +Changing either of these functions with an override won’t modify the behavior +of value_from().

+
+
+
+class celpy.c7nlib.C7NContext(filter: Any)[source]
+

Saves current C7N filter for use by functions in this module.

+

This is essential for making C7N filter available to some of these functions.

+
with C7NContext(filter):
+    cel_prgm.evaluate(cel_activation)
 
-

Here’s the essential rule:

-
x//y * y + x%y == x
+
+
+__init__(filter: Any) None[source]
+
+ +
+
+__repr__() str[source]
+

Return repr(self).

+
+ +
+
+__enter__() None[source]
+
+ +
+
+__exit__(exc_type: Type[BaseException] | None, exc_value: BaseException | None, traceback: TracebackType | None) None[source]
+
+ +
+ +
+
+celpy.c7nlib.key(source: ListType, target: StringType) BoolType | BytesType | DoubleType | DurationType | IntType | ListType | MapType | None | StringType | TimestampType | UintType[source]
+

The C7N shorthand tag:Name doesn’t translate well to CEL. It extracts a single value +from a sequence of objects with a {"Key": x, "Value": y} structure; specifically, +the value for y when x == "Name".

+

This function locate a particular “Key”: target within a list of {“Key”: x, “Value”, y} items, +returning the y value if one is found, null otherwise.

+

In effect, the key() function:

+
resource["Tags"].key("Name")["Value"]
 
-

However. Python // truncates toward negative infinity. Go / truncates toward zero.

-

To get Go-like behavior, we need to use absolute values and restore the signs later.

-
x_sign = -1 if x < 0 else +1
-go_mod = x_sign * (abs(x) % abs(y))
-return go_mod
+

is somewhat like:

+
resource["Tags"].filter(x, x["Key"] == "Name")[0]
 
-
-
-

Timzone Details

-

An implementation may have additional timezone names that must be injected into -the pendulum processing. (Formerly dateutil.gettz().)

-

For example, there may be the following sequence:

-
    -
  1. A lowercase match for an alias or an existing timezone.

  2. -
  3. A titlecase match for an existing timezone.

  4. -
  5. The fallback, which is a +/-HH:MM string.

  6. -
-
+

But the key() function doesn’t raise an exception if the key is not found, +instead it returns None.

+

We might want to generalize this into a first() reduction macro. +resource["Tags"].first(x, x["Key"] == "Name" ? x["Value"] : null, null) +This macro returns the first non-null value or the default (which can be null.)

+ +
-
-celpy.celtypes.type_matched(method: Callable[[Any, Any], Any]) Callable[[Any, Any], Any][source]
-

Decorates a method to assure the “other” value has the same type.

+
+celpy.c7nlib.glob(text: StringType, pattern: StringType) BoolType[source]
+

Compare a string with a pattern.

+

While "*.py".glob(some_string) seems logical because the pattern the more persistent object, +this seems to cause confusion.

+

We use some_string.glob("*.py") to express a regex-like rule. This parallels the CEL +.matches() method.

+

We also support glob(some_string, "*.py").

-
+
+celpy.c7nlib.difference(left: ListType, right: ListType) BoolType[source]
+

Compute the difference between two lists. This is ordered set difference: left - right. +It’s true if the result is non-empty: there is an item in the left, not present in the right. +It’s false if the result is empty: the lists are the same.

+
+ +
+
+celpy.c7nlib.intersect(left: ListType, right: ListType) BoolType[source]
+

Compute the intersection between two lists. +It’s true if the result is non-empty: there is an item in both lists. +It’s false if the result is empty: there is no common item between the lists.

+
+ +
+
+celpy.c7nlib.normalize(string: StringType) StringType[source]
+

Normalize a string.

+
+ +
+
+celpy.c7nlib.unique_size(collection: ListType) IntType[source]
+

Unique size of a list

+
+ +
+
+class celpy.c7nlib.IPv4Network(address, strict=True)[source]
+
+
+__contains__(other)[source]
+
+ +
+
+contains(other)
+
+ +
+ +
+
+celpy.c7nlib.parse_cidr(value: str) None | IPv4Network | IPv4Address[source]
+

Process cidr ranges.

+

This is a union of types outside CEL.

+

It appears to be Union[None, IPv4Network, ipaddress.IPv4Address]

+
+ +
+
+celpy.c7nlib.size_parse_cidr(value: StringType) IntType | None[source]
+

CIDR prefixlen value

+
+ +
+
+class celpy.c7nlib.ComparableVersion(version: str)[source]
+

The old LooseVersion could fail on comparing present strings, used +in the value as shorthand for certain options.

+

The new Version doesn’t fail as easily.

+
+
+__eq__(other: object) bool[source]
+

Return self==value.

+
+ +
+
+__hash__ = None
+
+ +
+ +
+
+celpy.c7nlib.version(value: StringType) BoolType | BytesType | DoubleType | DurationType | IntType | ListType | MapType | None | StringType | TimestampType | UintType[source]
+
+ +
+
+celpy.c7nlib.present(value: StringType) BoolType | BytesType | DoubleType | DurationType | IntType | ListType | MapType | None | StringType | TimestampType | UintType[source]
+
+ +
+
+celpy.c7nlib.absent(value: StringType) BoolType | BytesType | DoubleType | DurationType | IntType | ListType | MapType | None | StringType | TimestampType | UintType[source]
+
+ +
+
+celpy.c7nlib.text_from(url: StringType) BoolType | BytesType | DoubleType | DurationType | IntType | ListType | MapType | None | StringType | TimestampType | UintType[source]
+

Read raw text from a URL. This can be expanded to accept S3 or other URL’s.

+
+ +
+
+celpy.c7nlib.parse_text(source_text: StringType, format: StringType) BoolType | BytesType | DoubleType | DurationType | IntType | ListType | MapType | None | StringType | TimestampType | UintType[source]
+

Parse raw text using a given format.

+
+ +
+
+celpy.c7nlib.value_from(url: StringType, format: StringType | None = None) BoolType | BytesType | DoubleType | DurationType | IntType | ListType | MapType | None | StringType | TimestampType | UintType[source]
+

Read values from a URL.

+

First, do text_from() to read the source. +Then, do parse_text() to parse the source, if needed.

+

This makes the format optional, and deduces it from the URL’s path information.

+

C7N will generally replace this with a function +that leverages a more sophisticated c7n.resolver.ValuesFrom.

+
+ +
+
+celpy.c7nlib.jmes_path(source_data: BoolType | BytesType | DoubleType | DurationType | IntType | ListType | MapType | None | StringType | TimestampType | UintType, path_source: StringType) BoolType | BytesType | DoubleType | DurationType | IntType | ListType | MapType | None | StringType | TimestampType | UintType[source]
+

Apply JMESPath to an object read from from a URL.

+
+ +
+
+celpy.c7nlib.jmes_path_map(source_data: ListType, path_source: StringType) ListType[source]
+

Apply JMESPath to a each object read from from a URL. +This is for ndjson, nljson and jsonl files.

+
+ +
+
+celpy.c7nlib.marked_key(source: ListType, target: StringType) BoolType | BytesType | DoubleType | DurationType | IntType | ListType | MapType | None | StringType | TimestampType | UintType[source]
+

Examines a list of {“Key”: text, “Value”: text} mappings +looking for the given Key value.

+

Parses a message:action@action_date value into a mapping +{“message”: message, “action”: action, “action_date”: action_date}

+

If no Key or no Value or the Value isn’t the right structure, +the result is a null.

+
+ +
+
+celpy.c7nlib.image(resource: MapType) BoolType | BytesType | DoubleType | DurationType | IntType | ListType | MapType | None | StringType | TimestampType | UintType[source]
+

Reach into C7N to get the image details for this EC2 or ASG resource.

+

Minimally, the creation date is transformed into a CEL timestamp. +We may want to slightly generalize this to json_to_cell() the entire Image object.

+

The following may be usable, but it seems too complex:

+
C7N.filter.prefetch_instance_images(C7N.policy.resources)
+image = C7N.filter.get_instance_image(resource["ImageId"])
+return json_to_cel(image)
+
+
+
+ +
+
+celpy.c7nlib.get_raw_metrics(request: MapType) BoolType | BytesType | DoubleType | DurationType | IntType | ListType | MapType | None | StringType | TimestampType | UintType[source]
+

Reach into C7N and make a statistics request using the current C7N filter object.

+

The request parameter is the request object that is passed through to AWS via +the current C7N filter’s manager. The request is a Mapping with the following keys and values:

+
get_raw_metrics({
+    "Namespace": "AWS/EC2",
+    "MetricName": "CPUUtilization",
+    "Dimensions": {"Name": "InstanceId", "Value": resource.InstanceId},
+    "Statistics": ["Average"],
+    "StartTime": now - duration("4d"),
+    "EndTime": now,
+    "Period": duration("86400s")
+})
+
+
+

The request is passed through to AWS more-or-less directly. The result is a CEL +list of values for then requested statistic. A .map() macro +can be used to compute additional details. An .exists() macro can filter the +data to look for actionable values.

+

We would prefer to refactor C7N and implement this with code something like this:

+
C7N.filter.prepare_query(C7N.policy.resources)
+data = C7N.filter.get_resource_statistics(client, resource)
+return json_to_cel(data)
+
+
+
+ +
+
+celpy.c7nlib.get_metrics(resource: MapType, request: MapType) BoolType | BytesType | DoubleType | DurationType | IntType | ListType | MapType | None | StringType | TimestampType | UintType[source]
+

Reach into C7N and make a statistics request using the current C7N filter.

+

This builds a request object that is passed through to AWS via the get_raw_metrics() +function.

+

The request parameter is a Mapping with the following keys and values:

+
resource.get_metrics({"MetricName": "CPUUtilization", "Statistic": "Average",
+    "StartTime": now - duration("4d"), "EndTime": now, "Period": duration("86400s")}
+    ).exists(m, m < 30)
+
+
+

The namespace is derived from the C7N.policy. The dimensions are derived from +the C7N.fiter.model.

+
+ +
+
+celpy.c7nlib.get_raw_health_events(request: MapType) BoolType | BytesType | DoubleType | DurationType | IntType | ListType | MapType | None | StringType | TimestampType | UintType[source]
+

Reach into C7N and make a health-events request using the current C7N filter.

+

The request parameter is the filter object that is passed through to AWS via +the current C7N filter’s manager. The request is a List of AWS health events.

+
get_raw_health_events({
+    "services": ["ELASTICFILESYSTEM"],
+    "regions": ["us-east-1", "global"],
+    "eventStatusCodes": ['open', 'upcoming'],
+})
+
+
+
+ +
+
+celpy.c7nlib.get_health_events(resource: MapType, statuses: List[BoolType | BytesType | DoubleType | DurationType | IntType | ListType | MapType | None | StringType | TimestampType | UintType] | None = None) BoolType | BytesType | DoubleType | DurationType | IntType | ListType | MapType | None | StringType | TimestampType | UintType[source]
+

Reach into C7N and make a health-event request using the current C7N filter.

+

This builds a request object that is passed through to AWS via the get_raw_health_events() +function.

+
+ +
+ +

Reach into C7N and make a get_related_ids() request using the current C7N filter.

+
+ +
+ +

Reach into C7N and make a get_related_sgs() request using the current C7N filter.

+
+ +
+ +

Reach into C7N and make a get_related_subnets() request using the current C7N filter.

+
+ +
+ +

Reach into C7N and make a get_related_nat_gateways() request using the current C7N filter.

+
+ +
+ +

Reach into C7N and make a get_related_igws() request using the current C7N filter.

+
+ +
+ +

Reach into C7N and make a get_related_security_configs() request using the current C7N filter.

+
+ +
+ +

Reach into C7N and make a get_related_vpc() request using the current C7N filter.

+
+ +
+ +

Reach into C7N and make a get_related_kms_keys() request using the current C7N filter.

+
+ +
+
+celpy.c7nlib.security_group(security_group_id: MapType) BoolType | BytesType | DoubleType | DurationType | IntType | ListType | MapType | None | StringType | TimestampType | UintType[source]
+

Reach into C7N and make a get_related() request using the current C7N filter to get +the security group.

+
+ +
+
+celpy.c7nlib.subnet(subnet_id: BoolType | BytesType | DoubleType | DurationType | IntType | ListType | MapType | None | StringType | TimestampType | UintType) BoolType | BytesType | DoubleType | DurationType | IntType | ListType | MapType | None | StringType | TimestampType | UintType[source]
+

Reach into C7N and make a get_related() request using the current C7N filter to get +the subnet.

+
+ +
+
+celpy.c7nlib.flow_logs(resource: MapType) BoolType | BytesType | DoubleType | DurationType | IntType | ListType | MapType | None | StringType | TimestampType | UintType[source]
+

Reach into C7N and locate the flow logs using the current C7N filter.

+
+ +
+
+celpy.c7nlib.vpc(vpc_id: BoolType | BytesType | DoubleType | DurationType | IntType | ListType | MapType | None | StringType | TimestampType | UintType) BoolType | BytesType | DoubleType | DurationType | IntType | ListType | MapType | None | StringType | TimestampType | UintType[source]
+

Reach into C7N and make a get_related() request using the current C7N filter to get +the VPC details.

+
+ +
+
+celpy.c7nlib.subst(jmes_path: StringType) StringType[source]
+

Reach into C7N and build a set of substitutions to replace text in a JMES path.

+

This is based on how c7n.resolver.ValuesFrom works. There are +two possible substitution values:

+
    +
  • account_id

  • +
  • region

  • +
+
+
Parameters:
+

jmes_path – the source

+
+
Returns:
+

A JMES with values replaced.

+
+
+
+ +
+
+celpy.c7nlib.credentials(resource: MapType) BoolType | BytesType | DoubleType | DurationType | IntType | ListType | MapType | None | StringType | TimestampType | UintType[source]
+

Reach into C7N and make a get_related() request using the current C7N filter to get +the IAM-role credential details.

+

See c7n.resources.iam.CredentialReport filter. +The get_credential_report() function does what we need.

+
+ +
+
+celpy.c7nlib.kms_alias(vpc_id: BoolType | BytesType | DoubleType | DurationType | IntType | ListType | MapType | None | StringType | TimestampType | UintType) BoolType | BytesType | DoubleType | DurationType | IntType | ListType | MapType | None | StringType | TimestampType | UintType[source]
+

Reach into C7N and make a get_matching_aliases() request using the current C7N filter to get +the alias.

+

See c7n.resources.kms.ResourceKmsKeyAlias. +The get_matching_aliases() function does what we need.

+
+ +
+
+celpy.c7nlib.kms_key(key_id: BoolType | BytesType | DoubleType | DurationType | IntType | ListType | MapType | None | StringType | TimestampType | UintType) BoolType | BytesType | DoubleType | DurationType | IntType | ListType | MapType | None | StringType | TimestampType | UintType[source]
+

Reach into C7N and make a get_related() request using the current C7N filter to get +the key. We’re looking for the c7n.resources.kms.Key resource manager to get the related key.

+
+ +
+
+celpy.c7nlib.resource_schedule(tag_value: BoolType | BytesType | DoubleType | DurationType | IntType | ListType | MapType | None | StringType | TimestampType | UintType) BoolType | BytesType | DoubleType | DurationType | IntType | ListType | MapType | None | StringType | TimestampType | UintType[source]
+

Reach into C7N and use the the c7n.filters.offhours.ScheduleParser class +to examine the tag’s value, the current time, and return a True/False. +This parser is the parser value of the c7n.filters.offhours.Time filter class. +Using the filter’s parser.parse(value) provides needed structure.

+

The filter.parser.parse(value) will transform text of the Tag value +into a dictionary. This is further transformed to something we can use in CEL.

+

From this

+
off=[(M-F,21),(U,18)];on=[(M-F,6),(U,10)];tz=pt
+
+
+

C7N ScheduleParser produces this

+
{
+  off: [
+    { days: [1, 2, 3, 4, 5], hour: 21 },
+    { days: [0], hour: 18 }
+  ],
+  on: [
+    { days: [1, 2, 3, 4, 5], hour: 6 },
+    { days: [0], hour: 10 }
+  ],
+  tz: "pt"
+}
+
+
+

For CEL, we need this

+
{
+  off: [
+    { days: [1, 2, 3, 4, 5], hour: 21, tz: "pt" },
+    { days: [0], hour: 18, tz: "pt" }
+  ],
+  on: [
+    { days: [1, 2, 3, 4, 5], hour: 6, tz: "pt" },
+    { days: [0], hour: 10, tz: "pt" }
+  ],
+}
+
+
+

This lets a CEL expression use

+
key("maid_offhours").resource_schedule().off.exists(s,
+    now.getDayOfWeek(s.tz) in s.days && now.getHour(s.tz) == s.hour)
+
+
+
+ +
+
+celpy.c7nlib.get_accounts(resource: MapType) BoolType | BytesType | DoubleType | DurationType | IntType | ListType | MapType | None | StringType | TimestampType | UintType[source]
+

Reach into C7N filter and get accounts for a given resource. +Used by resources like AMI’s, log-groups, ebs-snapshot, etc.

+
+ +
+
+celpy.c7nlib.get_vpcs(resource: MapType) BoolType | BytesType | DoubleType | DurationType | IntType | ListType | MapType | None | StringType | TimestampType | UintType[source]
+

Reach into C7N filter and get vpcs for a given resource. +Used by resources like AMI’s, log-groups, ebs-snapshot, etc.

+
+ +
+
+celpy.c7nlib.get_vpces(resource: MapType) BoolType | BytesType | DoubleType | DurationType | IntType | ListType | MapType | None | StringType | TimestampType | UintType[source]
+

Reach into C7N filter and get vpces for a given resource. +Used by resources like AMI’s, log-groups, ebs-snapshot, etc.

+
+ +
+
+celpy.c7nlib.get_orgids(resource: MapType) BoolType | BytesType | DoubleType | DurationType | IntType | ListType | MapType | None | StringType | TimestampType | UintType[source]
+

Reach into C7N filter and get orgids for a given resource. +Used by resources like AMI’s, log-groups, ebs-snapshot, etc.

+
+ +
+
+celpy.c7nlib.get_endpoints(resource: MapType) BoolType | BytesType | DoubleType | DurationType | IntType | ListType | MapType | None | StringType | TimestampType | UintType[source]
+

For sns resources

+
+ +
+
+celpy.c7nlib.get_protocols(resource: MapType) BoolType | BytesType | DoubleType | DurationType | IntType | ListType | MapType | None | StringType | TimestampType | UintType[source]
+

For sns resources

+
+ +
+
+celpy.c7nlib.get_key_policy(resource: MapType) BoolType | BytesType | DoubleType | DurationType | IntType | ListType | MapType | None | StringType | TimestampType | UintType[source]
+

For kms resources

+
+ +
+
+celpy.c7nlib.get_resource_policy(resource: MapType) BoolType | BytesType | DoubleType | DurationType | IntType | ListType | MapType | None | StringType | TimestampType | UintType[source]
+

Reach into C7N filter and get the resource policy for a given resource. +Used by resources like AMI’s, log-groups, ebs-snapshot, etc.

+
+ +
+
+celpy.c7nlib.describe_subscription_filters(resource: MapType) BoolType | BytesType | DoubleType | DurationType | IntType | ListType | MapType | None | StringType | TimestampType | UintType[source]
+

For log-groups resources.

+
+ +
+
+celpy.c7nlib.describe_db_snapshot_attributes(resource: MapType) BoolType | BytesType | DoubleType | DurationType | IntType | ListType | MapType | None | StringType | TimestampType | UintType[source]
+

For rds-snapshot and ebs-snapshot resources

+
+ +
+
+celpy.c7nlib.arn_split(arn: StringType, field: StringType) BoolType | BytesType | DoubleType | DurationType | IntType | ListType | MapType | None | StringType | TimestampType | UintType[source]
+

Parse an ARN, removing a partivular field. +The field name must one one of +“partition”, “service”, “region”, “account-id”, “resource-type”, “resource-id” +In the case of a resource-type/resource-id path, this will be a “resource-id” value, +and there will be no “resource-type”.

+

Examples formats

+
+

arn:partition:service:region:account-id:resource-id

+

arn:partition:service:region:account-id:resource-type/resource-id

+

arn:partition:service:region:account-id:resource-type:resource-id

+
+
+ +
+
+celpy.c7nlib.all_images() BoolType | BytesType | DoubleType | DurationType | IntType | ListType | MapType | None | StringType | TimestampType | UintType[source]
+

Depends on CELFilter._pull_ec2_images() and CELFilter._pull_asg_images()

+

See c7n.resources.ami.ImageUnusedFilter

+
+ +
+
+celpy.c7nlib.all_snapshots() BoolType | BytesType | DoubleType | DurationType | IntType | ListType | MapType | None | StringType | TimestampType | UintType[source]
+

Depends on CELFilter._pull_asg_snapshots() +and CELFilter._pull_ami_snapshots()

+

See c7n.resources.ebs.SnapshotUnusedFilter

+
+ +
+
+celpy.c7nlib.all_launch_configuration_names() BoolType | BytesType | DoubleType | DurationType | IntType | ListType | MapType | None | StringType | TimestampType | UintType[source]
+

Depends on CELFilter.manager.get_launch_configuration_names()

+

See c7n.resources.asg.UnusedLaunchConfig

+
+ +
+
+celpy.c7nlib.all_service_roles() BoolType | BytesType | DoubleType | DurationType | IntType | ListType | MapType | None | StringType | TimestampType | UintType[source]
+

Depends on CELFilter.service_role_usage()

+

See c7n.resources.iam.UnusedIamRole

+
+ +
+
+celpy.c7nlib.all_instance_profiles() BoolType | BytesType | DoubleType | DurationType | IntType | ListType | MapType | None | StringType | TimestampType | UintType[source]
+

Depends on CELFilter.instance_profile_usage()

+

See c7n.resources.iam.UnusedInstanceProfiles

+
+ +
+
+celpy.c7nlib.all_dbsubenet_groups() BoolType | BytesType | DoubleType | DurationType | IntType | ListType | MapType | None | StringType | TimestampType | UintType[source]
+

Depends on CELFilter.get_dbsubnet_group_used()

+

See c7n.resources.rds.UnusedRDSSubnetGroup

+
+ +
+
+celpy.c7nlib.all_scan_groups() BoolType | BytesType | DoubleType | DurationType | IntType | ListType | MapType | None | StringType | TimestampType | UintType[source]
+

Depends on CELFilter.scan_groups()

+

See c7n.resources.vpc.UnusedSecurityGroup

+
+ +
+
+celpy.c7nlib.get_access_log(resource: MapType) BoolType | BytesType | DoubleType | DurationType | IntType | ListType | MapType | None | StringType | TimestampType | UintType[source]
+

Depends on CELFilter.resources()

+

See c7n.resources.elb.IsNotLoggingFilter and +c7n.resources.elb.IsLoggingFilter.

+
+ +
+
+celpy.c7nlib.get_load_balancer(resource: MapType) BoolType | BytesType | DoubleType | DurationType | IntType | ListType | MapType | None | StringType | TimestampType | UintType[source]
+

Depends on CELFilter.resources()

+

See c7n.resources.appelb.IsNotLoggingFilter and +c7n.resources.appelb.IsLoggingFilter.

+
+ +
+
+celpy.c7nlib.shield_protection(resource: MapType) BoolType | BytesType | DoubleType | DurationType | IntType | ListType | MapType | None | StringType | TimestampType | UintType[source]
+

Depends on the c7n.resources.shield.IsShieldProtected.process() method. +This needs to be refactored and renamed to avoid collisions with other process() variants.

+

Applies to most resource types.

+
+ +
+
+celpy.c7nlib.shield_subscription(resource: MapType) BoolType | BytesType | DoubleType | DurationType | IntType | ListType | MapType | None | StringType | TimestampType | UintType[source]
+

Depends on c7n.resources.account.ShieldEnabled.process() method. +This needs to be refactored and renamed to avoid collisions with other process() variants.

+

Applies to account resources only.

+
+ +
+
+celpy.c7nlib.web_acls(resource: MapType) BoolType | BytesType | DoubleType | DurationType | IntType | ListType | MapType | None | StringType | TimestampType | UintType[source]
+

Depends on c7n.resources.cloudfront.IsWafEnabled.process() method. +This needs to be refactored and renamed to avoid collisions with other process() variants.

+
+ +
+
+class celpy.c7nlib.C7N_Interpreted_Runner(environment: Environment, ast: Tree, functions: Dict[str, Callable[[...], BoolType | BytesType | DoubleType | DurationType | IntType | ListType | MapType | None | StringType | TimestampType | UintType | CELEvalError | Type[BoolType] | Type[BytesType] | Type[DoubleType] | Type[DurationType] | Type[IntType] | Type[ListType] | Type[MapType] | Callable[[...], None] | Type[StringType] | Type[TimestampType] | Type[TypeType] | Type[UintType] | Type[PackageType] | Type[MessageType]]] | None = None)[source]
+

Extends the Evaluation to introduce the C7N CELFilter instance into the evaluation.

+

The variable is global to allow the functions to have the simple-looking argument +values that CEL expects. This allows a function in this module to reach outside CEL for +access to C7N’s caches.

+
+
+evaluate(context: Mapping[str, BoolType | BytesType | DoubleType | DurationType | IntType | ListType | MapType | None | StringType | TimestampType | UintType | CELEvalError | Type[BoolType] | Type[BytesType] | Type[DoubleType] | Type[DurationType] | Type[IntType] | Type[ListType] | Type[MapType] | Callable[[...], None] | Type[StringType] | Type[TimestampType] | Type[TypeType] | Type[UintType] | Type[PackageType] | Type[MessageType] | NameContainer | CELFunction], filter: Any | None = None) BoolType | BytesType | DoubleType | DurationType | IntType | ListType | MapType | None | StringType | TimestampType | UintType[source]
+

Given variable definitions in the celpy.evaluation.Context, evaluate the given AST and return the resulting value.

+

Generally, this should raise an CELEvalError for most kinds of ordinary problems. +It may raise an CELUnsupportedError for future features that aren’t fully implemented. +Any Python exception reflects a serious problem.

+
+ +
+
+__abstractmethods__ = frozenset({})
+
+ +
+ +
+
+

celparser

+

A Facade around the CEL parser.

+

The Parser is an instance of the lark.Lark class.

+

The grammar is in the cel.lark file.

+

For more information on CEL syntax, see the following:

+ +

Example:

+
>>> from celpy.celparser import CELParser
+>>> p = CELParser()
+>>> text2 = 'type(null)'
+>>> ast2 = p.parse(text2)
+>>> print(ast2.pretty().replace("   ","   "))
+expr
+  conditionalor
+    conditionaland
+      relation
+        addition
+          multiplication
+            unary
+              member
+                primary
+                  ident_arg
+                    type
+                    exprlist
+                      expr
+                        conditionalor
+                          conditionaland
+                            relation
+                              addition
+                                multiplication
+                                  unary
+                                    member
+                                      primary
+                                        literal    null
+
+
+
+
+exception celpy.celparser.CELParseError(*args: Any, line: int | None = None, column: int | None = None)[source]
+

A syntax error in the CEL expression.

+
+
+__init__(*args: Any, line: int | None = None, column: int | None = None) None[source]
+
+ +
+ +
+
+class celpy.celparser.CELParser(tree_class: type = <class 'lark.tree.Tree'>)[source]
+

Creates a Lark parser with the required options.

+
+

Important

+

Singleton

+

There is one CEL_PARSER instance created by this class. +This is an optimization for environments like C7N where numerous +CEL expressions may parsed.

+
CELParse.CEL_PARSER = None
+
+
+

Is required to create another parser instance. +This is commonly required in test environments.

+
+

This is also an Adapter for the CEL parser to provide pleasant +syntax error messages.

+
+
+CEL_PARSER: Lark | None = None
+
+ +
+
+__init__(tree_class: type = <class 'lark.tree.Tree'>) None[source]
+
+ +
+
+static ambiguous_literals(t: Token) Token[source]
+

Resolve a grammar ambiguity between identifiers and literals

+
+ +
+
+parse(text: str) Tree[source]
+
+ +
+
+error_text(message: str, line: int | None = None, column: int | None = None) str[source]
+
+ +
+ +
+
+class celpy.celparser.DumpAST[source]
+

Dump a CEL AST creating a close approximation to the original source.

+
+
+classmethod display(ast: Tree) str[source]
+
+ +
+
+__init__() None[source]
+
+ +
+
+expr(tree: Tree) None[source]
+
+ +
+
+conditionalor(tree: Tree) None[source]
+
+ +
+
+conditionaland(tree: Tree) None[source]
+
+ +
+
+relation(tree: Tree) None[source]
+
+ +
+
+relation_lt(tree: Tree) None[source]
+
+ +
+
+relation_le(tree: Tree) None[source]
+
+ +
+
+relation_gt(tree: Tree) None[source]
+
+ +
+
+relation_ge(tree: Tree) None[source]
+
+ +
+
+relation_eq(tree: Tree) None[source]
+
+ +
+
+relation_ne(tree: Tree) None[source]
+
+ +
+
+relation_in(tree: Tree) None[source]
+
+ +
+
+addition(tree: Tree) None[source]
+
+ +
+
+addition_add(tree: Tree) None[source]
+
+ +
+
+addition_sub(tree: Tree) None[source]
+
+ +
+
+multiplication(tree: Tree) None[source]
+
+ +
+
+multiplication_mul(tree: Tree) None[source]
+
+ +
+
+multiplication_div(tree: Tree) None[source]
+
+ +
+
+multiplication_mod(tree: Tree) None[source]
+
+ +
+
+unary(tree: Tree) None[source]
+
+ +
+
+unary_not(tree: Tree) None[source]
+
+ +
+
+unary_neg(tree: Tree) None[source]
+
+ +
+
+member_dot(tree: Tree) None[source]
+
+ +
+
+member_dot_arg(tree: Tree) None[source]
+
+ +
+
+member_index(tree: Tree) None[source]
+
+ +
+
+member_object(tree: Tree) None[source]
+
+ +
+
+dot_ident_arg(tree: Tree) None[source]
+
+ +
+
+dot_ident(tree: Tree) None[source]
+
+ +
+
+ident_arg(tree: Tree) None[source]
+
+ +
+
+ident(tree: Tree) None[source]
+
+ +
+
+paren_expr(tree: Tree) None[source]
+
+ +
+
+list_lit(tree: Tree) None[source]
+
+ +
+
+map_lit(tree: Tree) None[source]
+
+ +
+
+exprlist(tree: Tree) None[source]
+
+ +
+
+fieldinits(tree: Tree) None[source]
+
+ +
+
+mapinits(tree: Tree) None[source]
+

Note reversed pop order for values and keys.

+
+ +
+
+literal(tree: Tree) None[source]
+
+ +
+
+__parameters__ = ()
+
+ +
+ +
+
+celpy.celparser.tree_dump(ast: Tree) str[source]
+

Dumps the AST to approximate the original source

+
+ +
+
+

celtypes

+

Provides wrappers over Python types to provide CEL semantics.

+

This can be used by a Python module to work with CEL-friendly values and CEL results.

+

Examples of distinctions between CEL and Python:

+
    +
  • Unlike Python bool, CEL BoolType won’t do some math.

  • +
  • CEL has int64 and uint64 subclasses of integer. These have specific ranges and +raise ValueError errors on overflow.

  • +
+

CEL types will raise ValueError for out-of-range values and TypeError +for operations they refuse. +The evaluation module can capture these exceptions and turn them into result values. +This can permit the logic operators to quietly silence them via “short-circuiting”.

+

In the normal course of events, CEL’s evaluator may attempt operations between a +CEL exception result and an instance of one of CEL types. +We rely on this leading to an ordinary Python TypeError to be raised to propogate +the error. Or. A logic operator may discard the error object.

+

The evaluation module extends these types with it’s own CELEvalError exception. +We try to keep that as a separate concern from the core operator implementations here. +We leverage Python features, which means raising exceptions when there is a problem.

+
+

Types

+

See https://github.com/google/cel-go/tree/master/common/types

+

These are the Go type definitions that are used by CEL:

+
    +
  • BoolType

  • +
  • BytesType

  • +
  • DoubleType

  • +
  • DurationType

  • +
  • IntType

  • +
  • ListType

  • +
  • MapType

  • +
  • NullType

  • +
  • StringType

  • +
  • TimestampType

  • +
  • TypeType

  • +
  • UintType

  • +
+

The above types are handled directly byt CEL syntax. +e.g., 42 vs. 42u vs. "42" vs. b"42" vs. 42..

+

We provide matching Python class names for each of these types. The Python type names +are subclasses of Python native types, allowing a client to transparently work with +CEL results. A Python host should be able to provide values to CEL that will be tolerated.

+

A type hint of Value unifies these into a common hint.

+

The CEL Go implementation also supports protobuf types:

+
    +
  • dpb.Duration

  • +
  • tpb.Timestamp

  • +
  • structpb.ListValue

  • +
  • structpb.NullValue

  • +
  • structpb.Struct

  • +
  • structpb.Value

  • +
  • wrapperspb.BoolValue

  • +
  • wrapperspb.BytesValue

  • +
  • wrapperspb.DoubleValue

  • +
  • wrapperspb.FloatValue

  • +
  • wrapperspb.Int32Value

  • +
  • wrapperspb.Int64Value

  • +
  • wrapperspb.StringValue

  • +
  • wrapperspb.UInt32Value

  • +
  • wrapperspb.UInt64Value

  • +
+

These types involve expressions like the following:

+
google.protobuf.UInt32Value{value: 123u}
+
+
+

In this case, the well-known protobuf name is directly visible as CEL syntax. +There’s a google package with the needed definitions.

+
+
+

Type Provider

+

A type provider can be bound to the environment, this will support additional types. +This appears to be a factory to map names of types to type classes.

+

Run-time type binding is shown by a CEL expression like the following:

+
TestAllTypes{single_uint32_wrapper: 432u}
+
+
+

The TestAllTypes is a protobuf type added to the CEL run-time. The syntax +is defined by this syntax rule:

+
member_object  : member "{" [fieldinits] "}"
+
+
+

The member is part of a type provider library, +either a standard protobuf definition or an extension. The field inits build +values for the protobuf object.

+

See https://github.com/google/cel-go/blob/master/test/proto3pb/test_all_types.proto +for the TestAllTypes protobuf definition that is registered as a type provider.

+

This expression will describes a Protobuf uint32 object.

+
+
+

Type Adapter

+

So far, it appears that a type adapter wraps existing Go or C++ types +with CEL-required methods. This seems like it does not need to be implemented +in Python.

+
+
+

Numeric Details

+

Integer division truncates toward zero.

+

The Go definition of modulus:

+
// Mod returns the floating-point remainder of x/y.
+// The magnitude of the result is less than y and its
+// sign agrees with that of x.
+
+
+

https://golang.org/ref/spec#Arithmetic_operators

+

“Go has the nice property that -a/b == -(a/b).”

+
 x     y     x / y     x % y
+ 5     3       1         2
+-5     3      -1        -2
+ 5    -3      -1         2
+-5    -3       1        -2
+
+
+

Python definition:

+
The modulo operator always yields a result
+with the same sign as its second operand (or zero);
+the absolute value of the result is strictly smaller than
+the absolute value of the second operand.
+
+
+

Here’s the essential rule:

+
x//y * y + x%y == x
+
+
+

However. Python // truncates toward negative infinity. Go / truncates toward zero.

+

To get Go-like behavior, we need to use absolute values and restore the signs later.

+
x_sign = -1 if x < 0 else +1
+go_mod = x_sign * (abs(x) % abs(y))
+return go_mod
+
+
+
+
+

Timzone Details

+

An implementation may have additional timezone names that must be injected into +the pendulum processing. (Formerly dateutil.gettz().)

+

For example, there may be the following sequence:

+
    +
  1. A lowercase match for an alias or an existing timezone.

  2. +
  3. A titlecase match for an existing timezone.

  4. +
  5. The fallback, which is a +/-HH:MM string.

  6. +
+
+
+
+celpy.celtypes.type_matched(method: Callable[[Any, Any], Any]) Callable[[Any, Any], Any][source]
+

Decorates a method to assure the “other” value has the same type.

+
+ +
+
celpy.celtypes.logical_condition(e: BoolType | BytesType | DoubleType | DurationType | IntType | ListType | MapType | None | StringType | TimestampType | UintType, x: BoolType | BytesType | DoubleType | DurationType | IntType | ListType | MapType | None | StringType | TimestampType | UintType, y: BoolType | BytesType | DoubleType | DurationType | IntType | ListType | MapType | None | StringType | TimestampType | UintType) BoolType | BytesType | DoubleType | DurationType | IntType | ListType | MapType | None | StringType | TimestampType | UintType[source]

CEL e ? x : y operator. Choose one of x or y. Exceptions in the unchosen expression are ignored.

@@ -596,14 +1865,16 @@

Timzone Details
celpy.celtypes.logical_not(x: BoolType | BytesType | DoubleType | DurationType | IntType | ListType | MapType | None | StringType | TimestampType | UintType) BoolType | BytesType | DoubleType | DurationType | IntType | ListType | MapType | None | StringType | TimestampType | UintType[source]
-

Native python not isn’t fully exposed for CEL types.

+

A function for native python not.

+

This could almost be logical_or = evaluation.boolean(operator.not_), +but the definition would expose Python’s notion of “truthiness”, which isn’t appropriate for CEL.

celpy.celtypes.logical_or(x: BoolType | BytesType | DoubleType | DurationType | IntType | ListType | MapType | None | StringType | TimestampType | UintType, y: BoolType | BytesType | DoubleType | DurationType | IntType | ListType | MapType | None | StringType | TimestampType | UintType) BoolType | BytesType | DoubleType | DurationType | IntType | ListType | MapType | None | StringType | TimestampType | UintType[source]
-

Native Python has a left-to-right rule: (True or y) is True, (False or y) is y. -CEL || is commutative with non-Boolean values, including errors. +

Native Python has a left-to-right rule: (True or y) is True, (False or y) is y. +CEL || is commutative with non-Boolean values, including errors. (x || false) is x, and (false || y) is y.

Example 1:

false || 1/0 != 0
@@ -615,14 +1886,14 @@ 

Timzone DetailsBoolType, we’ll create an TypeError that will become a CELEvalError.

class celpy.celtypes.BoolType(source: Any)[source]
-

Native Python permits unary operators on Booleans.

-

For CEL, We need to prevent -false from working.

+

Native Python permits all unary operators to work on bool objects.

+

For CEL, we need to prevent the CEL expression -false from working.

static __new__(cls: Type[BoolType], source: Any) BoolType[source]
@@ -1126,6 +2397,11 @@

Timzone Details +
+contains(item: BoolType | BytesType | DoubleType | DurationType | IntType | ListType | MapType | None | StringType | TimestampType | UintType) BoolType[source]
+

+
__hash__ = None
@@ -1184,12 +2460,23 @@

Timzone Details +
+get(key: Any, default: Any | None = None) BoolType | BytesType | DoubleType | DurationType | IntType | ListType | MapType | None | StringType | TimestampType | UintType[source]
+

There is no default provision in CEL, that’s a Python feature.

+

+
static valid_key_type(key: Any) bool[source]

Valid CEL key types. Plus native str for tokens in the source when evaluating e.f

+
+
+contains(item: BoolType | BytesType | DoubleType | DurationType | IntType | ListType | MapType | None | StringType | TimestampType | UintType) BoolType[source]
+
+
__hash__ = None
@@ -1264,6 +2551,11 @@

Timzone Details +
+contains(item: BoolType | BytesType | DoubleType | DurationType | IntType | ListType | MapType | None | StringType | TimestampType | UintType) BoolType[source]
+

+
@@ -1354,7 +2646,7 @@

Timzone Details
classmethod tz_name_lookup(tz_name: str) tzinfo | None[source]
-

The dateutil.tz.gettz() may be extended with additional aliases.

+

The pendulum parsing may be extended with additional aliases.

@@ -1573,54 +2865,45 @@

Timzone Details
-class celpy.celtypes.TypeType(value: Any = '')[source]
-

Annotation used to mark protobuf type objects. -We map these to CELTypes so that type name testing works.

-
-
-type_name_mapping = {'BOOL': <class 'celpy.celtypes.BoolType'>, 'BYTES': <class 'celpy.celtypes.BytesType'>, 'DOUBLE': <class 'celpy.celtypes.DoubleType'>, 'INT32': <class 'celpy.celtypes.IntType'>, 'INT64': <class 'celpy.celtypes.IntType'>, 'STRING': <class 'celpy.celtypes.StringType'>, 'UINT32': <class 'celpy.celtypes.UintType'>, 'UINT64': <class 'celpy.celtypes.UintType'>, 'bool': <class 'celpy.celtypes.BoolType'>, 'bytes': <class 'celpy.celtypes.BytesType'>, 'double': <class 'celpy.celtypes.DoubleType'>, 'google.protobuf.Any': <class 'celpy.celtypes.MessageType'>, 'google.protobuf.DoubleValue': <class 'celpy.celtypes.DoubleType'>, 'google.protobuf.Duration': <class 'celpy.celtypes.DurationType'>, 'google.protobuf.FloatValue': <class 'celpy.celtypes.DoubleType'>, 'google.protobuf.Int32Value': <class 'celpy.celtypes.IntType'>, 'google.protobuf.Int64Value': <class 'celpy.celtypes.IntType'>, 'google.protobuf.Timestamp': <class 'celpy.celtypes.TimestampType'>, 'google.protobuf.UInt32Value': <class 'celpy.celtypes.UintType'>, 'google.protobuf.UInt64Value': <class 'celpy.celtypes.UintType'>, 'google.protobuf.Value': <class 'celpy.celtypes.MessageType'>, 'google.protubuf.Any': <class 'celpy.celtypes.MessageType'>, 'int': <class 'celpy.celtypes.IntType'>, 'list': <class 'celpy.celtypes.ListType'>, 'list_type': <class 'celpy.celtypes.ListType'>, 'map': <class 'celpy.celtypes.MapType'>, 'map_type': <class 'celpy.celtypes.MapType'>, 'null_type': <class 'NoneType'>, 'string': <class 'celpy.celtypes.StringType'>, 'uint': <class 'celpy.celtypes.UintType'>}
-
- +class celpy.celtypes.TypeType(instance: Any)[source] +

This is primarily used as a function to extract type from an object or a type. +For consistence, we define it as a type so other types can extend it.

-
-__init__(value: Any = '') None[source]
-
- -
-
-__hash__ = None
+
+static __new__(typ, instance: Any) type[source]
-
-
-__eq__(other: Any) bool[source]
-

Return self==value.

-
-

-

evaluation

-

CEL Interpreter using the AST directly.

+

evaluation

+

Evaluates CEL expressions given an AST.

+

There are two implementations:

+
    +
  • Evaluator – interprets the AST directly.

  • +
  • Transpiler – transpiles the AST to Python, compiles the Python to create a code object, and then uses exec() to evaluate the code object.

  • +

The general idea is to map CEL operators to Python operators and push the real work off to Python objects defined by the celpy.celtypes module.

-

CEL operator “+” is implemented by “_+_” function. We map this to operator.add(). -This will then look for __add__() methods in the various celpy.celtypes.CELType +

CEL operator + is implemented by a "_+_" function. +We map this name to operator.add(). +This will then look for __add__() methods in the various celpy.celtypes types.

In order to deal gracefully with missing and incomplete data, -exceptions are turned into first-class Result objects. +checked exceptions are used. +A raised exception is turned into first-class celpy.celtypes.Result object. They’re not raised directly, but instead saved as part of the evaluation so that short-circuit operators can ignore the exceptions.

This means that Python exceptions like TypeError, IndexError, and KeyError are caught and transformed into CELEvalError objects.

-

The Resut type hint is a union of the various values that are encountered +

The celpy.celtypes.Result type hint is a union of the various values that are encountered during evaluation. It’s a union of the celpy.celtypes.CELTypes type and the CELEvalError exception.

Important

Debugging

-

If the os environment variable CEL_TRACE is set, then detailed tracing of methods is made available. +

If the OS environment variable CEL_TRACE is set, then detailed tracing of methods is made available. To see the trace, set the logging level for celpy.Evaluator to logging.DEBUG.

@@ -1777,7 +3060,7 @@

Timzone Details
Parameters:

-

In effect, this class is

-
Referent = Union[
-    Annotation,
-    celpy.celtypes.Value,
-    CELEvalError,
-    CELFunction,
-]
-
+
+

Note

+

Future Design

+

A Referent is (almost) a tuple[Annotation, NameContainer | None, Value | NotSetSentinel]. +The current implementation is stateful, because values are optional and may be added later. +The use of a special sentinel to indicate the value was not set is a little akward. +It’s not really a 3-tuple, because NameContainers don’t have values; they are a kind of value. +(None is a valid value, and can’t be used for this.)

+

It may be slightly simpler to use a union of two types: +tuple[Annotation] | tuple[Annotation, NameContainer | Value]. +One-tuples capture the Annotation for a name; two-tuples capture Annotation and Value (or subsidiary NameContainer).

-__init__(ref_to: Type[BoolType] | Type[BytesType] | Type[DoubleType] | Type[DurationType] | Type[IntType] | Type[ListType] | Type[MapType] | Callable[[...], None] | Type[StringType] | Type[TimestampType] | Type[TypeType] | Type[UintType] | Type[PackageType] | Type[MessageType] | Callable[[...], BoolType | BytesType | DoubleType | DurationType | IntType | ListType | MapType | None | StringType | TimestampType | UintType] | Type[FunctionType] | None = None) None[source]
+__init__(ref_to: TypeType | Callable[[...], BoolType | BytesType | DoubleType | DurationType | IntType | ListType | MapType | None | StringType | TimestampType | UintType] | Type[FunctionType] | None = None) None[source]
@@ -1901,9 +3286,15 @@

Timzone Details +
+__eq__(other: Any) bool[source]
+

Return self==value.

+

+
-property value: Type[BoolType] | Type[BytesType] | Type[DoubleType] | Type[DurationType] | Type[IntType] | Type[ListType] | Type[MapType] | Callable[[...], None] | Type[StringType] | Type[TimestampType] | Type[TypeType] | Type[UintType] | Type[PackageType] | Type[MessageType] | Callable[[...], BoolType | BytesType | DoubleType | DurationType | IntType | ListType | MapType | None | StringType | TimestampType | UintType] | Type[FunctionType] | BoolType | BytesType | DoubleType | DurationType | IntType | ListType | MapType | None | StringType | TimestampType | UintType | CELEvalError | Callable[[...], BoolType | BytesType | DoubleType | DurationType | IntType | ListType | MapType | None | StringType | TimestampType | UintType | CELEvalError | Type[BoolType] | Type[BytesType] | Type[DoubleType] | Type[DurationType] | Type[IntType] | Type[ListType] | Type[MapType] | Callable[[...], None] | Type[StringType] | Type[TimestampType] | Type[TypeType] | Type[UintType] | Type[PackageType] | Type[MessageType]] | NameContainer
+property value: TypeType | Callable[[...], BoolType | BytesType | DoubleType | DurationType | IntType | ListType | MapType | None | StringType | TimestampType | UintType] | Type[FunctionType] | BoolType | BytesType | DoubleType | DurationType | IntType | ListType | MapType | None | StringType | TimestampType | UintType | CELEvalError | Callable[[...], BoolType | BytesType | DoubleType | DurationType | IntType | ListType | MapType | None | StringType | TimestampType | UintType | CELEvalError | Type[BoolType] | Type[BytesType] | Type[DoubleType] | Type[DurationType] | Type[IntType] | Type[ListType] | Type[MapType] | Callable[[...], None] | Type[StringType] | Type[TimestampType] | Type[TypeType] | Type[UintType] | Type[PackageType] | Type[MessageType]] | NameContainer

The longest-path rule means we prefer NameContainer over any locally defined value. Otherwise, we’ll provide a value if there is one. Finally, we’ll provide the annotation if there’s no value. @@ -1915,6 +3306,11 @@

Timzone Detailsclone() Referent[source]

+
+
+__hash__ = None
+
+
@@ -1953,28 +3349,28 @@

Timzone DetailsStructure

+

A NameContainer is a mapping from names to Referent instances.

+

A Referent can be one of several things, including…

  • A NameContainer further down the path

  • An Annotation

  • -
  • An Annotation with a value.

  • +
  • An Annotation and a value

  • +
  • A CELFunction (In effect, an Annotation of CELFunction, and a value of the function implementation.)

-

Loading Names.

-

There are several “phases” to building the chain of NameContainer instances.

+

Life and Content

+

There are two phases to building the chain of NameContainer instances.

  1. The Activation creates the initial name : annotation bindings. Generally, the names are type names, like “int”, bound to celtypes.IntType. In some cases, the name is a future variable name, “resource”, bound to celtypes.MapType.

  2. -
  3. The Activation creates a second NameContainer that has variable names. -This has a reference back to the parent to resolve names that are types.

  4. +
  5. The Activation updates some variables to provide values.

-

This involves decomposing the paths of names to make a tree of nested NameContainers. +

A name is decomposed into a path to make a tree of nested NameContainers. Upper-level containers don’t (necessarily) have types or values – they’re merely NameContainer along the path to the target names.

-

Resolving Names.

+

Resolving Names

See https://github.com/google/cel-spec/blob/master/doc/langdef.md#name-resolution

There are three cases required in the Evaluator engine.

The evaluator’s ident_value() method resolves the identifier into the Referent.

-

Acceptance Test Case

+

Acceptance Test Cases

We have two names

@@ -2077,13 +3473,14 @@

Timzone Details
-static dict_find_name(some_dict: Dict[str, Referent], path: List[str]) BoolType | BytesType | DoubleType | DurationType | IntType | ListType | MapType | None | StringType | TimestampType | UintType | CELEvalError | Type[BoolType] | Type[BytesType] | Type[DoubleType] | Type[DurationType] | Type[IntType] | Type[ListType] | Type[MapType] | Callable[[...], None] | Type[StringType] | Type[TimestampType] | Type[TypeType] | Type[UintType] | Type[PackageType] | Type[MessageType][source]
-

Extension to navgiate into mappings, messages, and packages.

+static dict_find_name(some_dict: Dict[str, Referent] | Referent, path: Sequence[str]) Referent[source] +

Recursive navigation into mappings, messages, and packages. +These are not NameContainers (or Activations).

Parameters:
    -
  • some_dict – An instance of a MapType, MessageType, or PackageType.

  • -
  • path – names to follow into the structure.

  • +
  • some_dict – An instance of a MapType, MessageType, or PackageType.

  • +
  • path – sequence of names to follow into the structure.

Returns:
@@ -2094,9 +3491,11 @@

Timzone Details
-find_name(path: List[str]) NameContainer | BoolType | BytesType | DoubleType | DurationType | IntType | ListType | MapType | None | StringType | TimestampType | UintType | CELEvalError | Type[BoolType] | Type[BytesType] | Type[DoubleType] | Type[DurationType] | Type[IntType] | Type[ListType] | Type[MapType] | Callable[[...], None] | Type[StringType] | Type[TimestampType] | Type[TypeType] | Type[UintType] | Type[PackageType] | Type[MessageType][source]
+find_name(path: List[str]) Referent[source]

Find the name by searching down through nested packages or raise NotFound. -This is a kind of in-order tree walk of contained packages.

+Returns the Value associated with this Name.

+

This is a kind of in-order tree walk of contained packages.

+

The collaborator must choose the annotation or the value from the Referent.

@@ -2113,9 +3512,9 @@

Timzone Details -
  • A.B.a.b. (Search for “a” in paackage “A.B”; the “.b” is handled separately.)

  • -
  • A.a.b. (Search for “a” in paackage “A”; the “.b” is handled separately.)

  • -
  • (finally) a.b. (Search for “a” in paackage None; the “.b” is handled separately.)

  • +
  • A.B.a.b. (Search for “a” in package “A.B”; the “.b” is handled separately.)

  • +
  • A.a.b. (Search for “a” in package “A”; the “.b” is handled separately.)

  • +
  • (finally) a.b. (Search for “a” in package None; the “.b” is handled separately.)

  • To override this behavior, one can use .a.b; this name will only be attempted to be resolved in the root scope, i.e. as a.b.

    @@ -2123,21 +3522,24 @@

    Timzone DetailsNameContainer and all parents in the parent_iter() iterable. The first name we find in the parent sequence is the goal. -This is because values are first, type annotations are laast.

    +This is because values are first, type annotations are last.

    If we can’t find the identifier with given package target, truncate the package name from the end to create a new target and try again. This is a bottom-up look that favors the longest name.

    Parameters:
      -
    • package – Prefix string “name.name.name”

    • +
    • package – Prefix string “path.path.path”

    • name – The variable we’re looking for

    Returns:
    -

    Name resolution as a Rereferent, often a value, but maybe a package or an +

    Name resolution as a Rereferent, often a value, but maybe a package or an annotation.

    +
    Raises:
    +

    KeyError – if the name cannot be found in this NameContainer

    +

    @@ -2146,6 +3548,16 @@

    Timzone Detailsclone() NameContainer[source]
    +
    +
    +get(name: str, default: Referent | None = None) TypeType | Callable[[...], BoolType | BytesType | DoubleType | DurationType | IntType | ListType | MapType | None | StringType | TimestampType | UintType] | Type[FunctionType] | BoolType | BytesType | DoubleType | DurationType | IntType | ListType | MapType | None | StringType | TimestampType | UintType | CELEvalError | Callable[[...], BoolType | BytesType | DoubleType | DurationType | IntType | ListType | MapType | None | StringType | TimestampType | UintType | CELEvalError | Type[BoolType] | Type[BytesType] | Type[DoubleType] | Type[DurationType] | Type[IntType] | Type[ListType] | Type[MapType] | Callable[[...], None] | Type[StringType] | Type[TimestampType] | Type[TypeType] | Type[UintType] | Type[PackageType] | Type[MessageType]] | NameContainer[source]
    +

    Used by transpiled code to get values from a NameContainer of Referents.

    +
    +

    Important

    +

    This does not get a Referent, it gets a value.

    +
    +
    +
    __repr__() str[source]
    @@ -2166,23 +3578,25 @@

    Timzone Details
    -class celpy.evaluation.Activation(annotations: Mapping[str, Type[BoolType] | Type[BytesType] | Type[DoubleType] | Type[DurationType] | Type[IntType] | Type[ListType] | Type[MapType] | Callable[[...], None] | Type[StringType] | Type[TimestampType] | Type[TypeType] | Type[UintType] | Type[PackageType] | Type[MessageType] | Callable[[...], BoolType | BytesType | DoubleType | DurationType | IntType | ListType | MapType | None | StringType | TimestampType | UintType] | Type[FunctionType]] | None = None, package: str | None = None, vars: Mapping[str, BoolType | BytesType | DoubleType | DurationType | IntType | ListType | MapType | None | StringType | TimestampType | UintType | CELEvalError | Type[BoolType] | Type[BytesType] | Type[DoubleType] | Type[DurationType] | Type[IntType] | Type[ListType] | Type[MapType] | Callable[[...], None] | Type[StringType] | Type[TimestampType] | Type[TypeType] | Type[UintType] | Type[PackageType] | Type[MessageType] | NameContainer] | None = None, parent: Activation | None = None)[source]
    -

    Namespace with variable bindings and type name (“annotation”) bindings.

    +class celpy.evaluation.Activation(*, annotations: Mapping[str, TypeType | Callable[[...], BoolType | BytesType | DoubleType | DurationType | IntType | ListType | MapType | None | StringType | TimestampType | UintType] | Type[FunctionType]] | None = None, vars: Mapping[str, BoolType | BytesType | DoubleType | DurationType | IntType | ListType | MapType | None | StringType | TimestampType | UintType | CELEvalError | Type[BoolType] | Type[BytesType] | Type[DoubleType] | Type[DurationType] | Type[IntType] | Type[ListType] | Type[MapType] | Callable[[...], None] | Type[StringType] | Type[TimestampType] | Type[TypeType] | Type[UintType] | Type[PackageType] | Type[MessageType] | NameContainer | CELFunction] | None = None, functions: Mapping[str, Callable[[...], BoolType | BytesType | DoubleType | DurationType | IntType | ListType | MapType | None | StringType | TimestampType | UintType | CELEvalError | Type[BoolType] | Type[BytesType] | Type[DoubleType] | Type[DurationType] | Type[IntType] | Type[ListType] | Type[MapType] | Callable[[...], None] | Type[StringType] | Type[TimestampType] | Type[TypeType] | Type[UintType] | Type[PackageType] | Type[MessageType]]] | list[Callable[[...], BoolType | BytesType | DoubleType | DurationType | IntType | ListType | MapType | None | StringType | TimestampType | UintType | CELEvalError | Type[BoolType] | Type[BytesType] | Type[DoubleType] | Type[DurationType] | Type[IntType] | Type[ListType] | Type[MapType] | Callable[[...], None] | Type[StringType] | Type[TimestampType] | Type[TypeType] | Type[UintType] | Type[PackageType] | Type[MessageType]]] | None = None, package: str | None = None, based_on: Activation | None = None)[source] +

    Namespace with variable bindings and type name (“annotation”) bindings. +Additionally, the pool of functions and types are here, also.

    Life and Content

    An Activation is created by an Environment and contains the annotations (and a package name) from that Environment. Variables are loaded into the activation for evaluation.

    A nested Activation is created each time we evaluate a macro.

    -

    An Activation contains a NameContainer instance to resolve identifers. +

    An Activation contains a NameContainer instance to resolve identifiers. (This may be a needless distinction and the two classes could, perhaps, be combined.)

    +

    These names include variables as well as type names used for protobuf and the internal CEL type() function.

    Chaining/Nesting

    Activations can form a chain so locals are checked first. Activations can nest via macro evaluation, creating transient local variables.

    +

    Consider this CEL macro expression:

    ``"[2, 4, 6].map(n, n / 2)"``
     
    -

    means nested activations with n bound to 2, 4, and 6 respectively. -The resulting objects then form a resulting list.

    +

    This works via a nested activation with n bound to 2, 4, and 6 respectively.

    This is used by an Evaluator as follows:

    sub_activation: Activation = self.activation.nested_activation()
     sub_eval: Evaluator = self.sub_eval(sub_activation)
    @@ -2214,22 +3628,24 @@ 

    Timzone Details"namespace resolution should try to find the longest prefix for the evaluator."

    -

    Most names start with IDENT, but a primary can start with ..

    +

    Most names start with IDENT, but a primary can start with .. +A leading . changes the search order from most local first to root first.

    -__init__(annotations: Mapping[str, Type[BoolType] | Type[BytesType] | Type[DoubleType] | Type[DurationType] | Type[IntType] | Type[ListType] | Type[MapType] | Callable[[...], None] | Type[StringType] | Type[TimestampType] | Type[TypeType] | Type[UintType] | Type[PackageType] | Type[MessageType] | Callable[[...], BoolType | BytesType | DoubleType | DurationType | IntType | ListType | MapType | None | StringType | TimestampType | UintType] | Type[FunctionType]] | None = None, package: str | None = None, vars: Mapping[str, BoolType | BytesType | DoubleType | DurationType | IntType | ListType | MapType | None | StringType | TimestampType | UintType | CELEvalError | Type[BoolType] | Type[BytesType] | Type[DoubleType] | Type[DurationType] | Type[IntType] | Type[ListType] | Type[MapType] | Callable[[...], None] | Type[StringType] | Type[TimestampType] | Type[TypeType] | Type[UintType] | Type[PackageType] | Type[MessageType] | NameContainer] | None = None, parent: Activation | None = None) None[source]
    +__init__(*, annotations: Mapping[str, TypeType | Callable[[...], BoolType | BytesType | DoubleType | DurationType | IntType | ListType | MapType | None | StringType | TimestampType | UintType] | Type[FunctionType]] | None = None, vars: Mapping[str, BoolType | BytesType | DoubleType | DurationType | IntType | ListType | MapType | None | StringType | TimestampType | UintType | CELEvalError | Type[BoolType] | Type[BytesType] | Type[DoubleType] | Type[DurationType] | Type[IntType] | Type[ListType] | Type[MapType] | Callable[[...], None] | Type[StringType] | Type[TimestampType] | Type[TypeType] | Type[UintType] | Type[PackageType] | Type[MessageType] | NameContainer | CELFunction] | None = None, functions: Mapping[str, Callable[[...], BoolType | BytesType | DoubleType | DurationType | IntType | ListType | MapType | None | StringType | TimestampType | UintType | CELEvalError | Type[BoolType] | Type[BytesType] | Type[DoubleType] | Type[DurationType] | Type[IntType] | Type[ListType] | Type[MapType] | Callable[[...], None] | Type[StringType] | Type[TimestampType] | Type[TypeType] | Type[UintType] | Type[PackageType] | Type[MessageType]]] | list[Callable[[...], BoolType | BytesType | DoubleType | DurationType | IntType | ListType | MapType | None | StringType | TimestampType | UintType | CELEvalError | Type[BoolType] | Type[BytesType] | Type[DoubleType] | Type[DurationType] | Type[IntType] | Type[ListType] | Type[MapType] | Callable[[...], None] | Type[StringType] | Type[TimestampType] | Type[TypeType] | Type[UintType] | Type[PackageType] | Type[MessageType]]] | None = None, package: str | None = None, based_on: Activation | None = None) None[source]

    Create an Activation.

    -

    The annotations are loaded first. The variables are loaded second, and placed -in front of the annotations in the chain of name resolutions. Values come before -annotations.

    +

    The annotations are loaded first. The variables and their values are loaded second, and placed +in front of the annotations in the chain of name resolutions.

    +

    The Evaluator and the Transpiler use this to resolve identifiers into types, values, or functions.

    Parameters:
    • annotations – Variables and type annotations. Annotations are loaded first to serve as defaults to create a parent NameContainer.

    • -
    • package – The package name to assume as a prefix for name resolution.

    • vars – Variables and their values, loaded to update the NameContainer.

    • -
    • parent – A parent activation in the case of macro evaluations.

    • +
    • functions – functions and their implementation, loaded to update the NameContainer.

    • +
    • package – The package name to assume as a prefix for name resolution.

    • +
    • based_on – A foundational activation on which this is based.

    @@ -2243,66 +3659,66 @@

    Timzone Details
    -nested_activation(annotations: Mapping[str, Type[BoolType] | Type[BytesType] | Type[DoubleType] | Type[DurationType] | Type[IntType] | Type[ListType] | Type[MapType] | Callable[[...], None] | Type[StringType] | Type[TimestampType] | Type[TypeType] | Type[UintType] | Type[PackageType] | Type[MessageType] | Callable[[...], BoolType | BytesType | DoubleType | DurationType | IntType | ListType | MapType | None | StringType | TimestampType | UintType] | Type[FunctionType]] | None = None, vars: Mapping[str, BoolType | BytesType | DoubleType | DurationType | IntType | ListType | MapType | None | StringType | TimestampType | UintType | CELEvalError | Type[BoolType] | Type[BytesType] | Type[DoubleType] | Type[DurationType] | Type[IntType] | Type[ListType] | Type[MapType] | Callable[[...], None] | Type[StringType] | Type[TimestampType] | Type[TypeType] | Type[UintType] | Type[PackageType] | Type[MessageType] | NameContainer] | None = None) Activation[source]
    -

    Create a nested sub-Activation that chains to the current activation. -The sub-activations don’t have the same implied package context,

    +nested_activation(annotations: Mapping[str, TypeType | Callable[[...], BoolType | BytesType | DoubleType | DurationType | IntType | ListType | MapType | None | StringType | TimestampType | UintType] | Type[FunctionType]] | None = None, vars: Mapping[str, BoolType | BytesType | DoubleType | DurationType | IntType | ListType | MapType | None | StringType | TimestampType | UintType | CELEvalError | Type[BoolType] | Type[BytesType] | Type[DoubleType] | Type[DurationType] | Type[IntType] | Type[ListType] | Type[MapType] | Callable[[...], None] | Type[StringType] | Type[TimestampType] | Type[TypeType] | Type[UintType] | Type[PackageType] | Type[MessageType] | NameContainer | CELFunction] | None = None) Activation[source] +

    Create an Activation based on the current activation. +This new Activation will be seeded from the current activation’s +NameContainer.

    Parameters:
      -
    • annotations – Variable type annotations

    • -
    • vars – Variables with literals to be converted to the desired types.

    • +
    • annotations – Optional type definitions for the new local variables.

    • +
    • vars – Local variables to be added when creating this activation.

    Returns:
    -

    An Activation that chains to this Activation.

    +

    A subsidiary Activation that chains to this Activation.

    -resolve_variable(name: str) BoolType | BytesType | DoubleType | DurationType | IntType | ListType | MapType | None | StringType | TimestampType | UintType | CELEvalError | Type[BoolType] | Type[BytesType] | Type[DoubleType] | Type[DurationType] | Type[IntType] | Type[ListType] | Type[MapType] | Callable[[...], None] | Type[StringType] | Type[TimestampType] | Type[TypeType] | Type[UintType] | Type[PackageType] | Type[MessageType] | NameContainer[source]
    +resolve_variable(name: str) BoolType | BytesType | DoubleType | DurationType | IntType | ListType | MapType | None | StringType | TimestampType | UintType | Callable[[...], BoolType | BytesType | DoubleType | DurationType | IntType | ListType | MapType | None | StringType | TimestampType | UintType | CELEvalError | Type[BoolType] | Type[BytesType] | Type[DoubleType] | Type[DurationType] | Type[IntType] | Type[ListType] | Type[MapType] | Callable[[...], None] | Type[StringType] | Type[TimestampType] | Type[TypeType] | Type[UintType] | Type[PackageType] | Type[MessageType]] | NameContainer[source]

    Find the object referred to by the name.

    An Activation usually has a chain of NameContainers to be searched.

    A variable can refer to an annotation and/or a value and/or a nested container. Most of the time, we want the value attribute of the Referent. This can be a Result (a Union[Value, CelType])

    +

    There’s a subtle difference between a variable without an annotation, +and a variable with an annotation, but without a value.

    -
    -__repr__() str[source]
    -

    Return repr(self).

    -
    - +
    +resolve_function(name: str) Callable[[...], BoolType | BytesType | DoubleType | DurationType | IntType | ListType | MapType | None | StringType | TimestampType | UintType | CELEvalError | Type[BoolType] | Type[BytesType] | Type[DoubleType] | Type[DurationType] | Type[IntType] | Type[ListType] | Type[MapType] | Callable[[...], None] | Type[StringType] | Type[TimestampType] | Type[TypeType] | Type[UintType] | Type[PackageType] | Type[MessageType]] | TypeType[source]
    +

    A short-cut to find functions without looking at Variables first.

    -
    -
    -class celpy.evaluation.FindIdent[source]
    -

    Locate the ident token at the bottom of an AST.

    -

    This is needed to find the bind variable for macros.

    -

    It works by doing a “visit” on the entire tree, but saving -the details of the ident nodes only.

    -
    -__init__() None[source]
    -
    +
    +__getattr__(name: str) BoolType | BytesType | DoubleType | DurationType | IntType | ListType | MapType | None | StringType | TimestampType | UintType | Callable[[...], BoolType | BytesType | DoubleType | DurationType | IntType | ListType | MapType | None | StringType | TimestampType | UintType | CELEvalError | Type[BoolType] | Type[BytesType] | Type[DoubleType] | Type[DurationType] | Type[IntType] | Type[ListType] | Type[MapType] | Callable[[...], None] | Type[StringType] | Type[TimestampType] | Type[TypeType] | Type[UintType] | Type[PackageType] | Type[MessageType]] | NameContainer[source]
    +

    Handle activation.name in transpiled code (or activation.get('name')).

    +

    If the name is not in the Activation with a value, a NameError exception must be raised.

    +

    Note that Activation.resolve_variable() depends on NameContainer.find_name(). +The NameContainer.find_name() method also find the value.

    +

    This is – perhaps – less than optimal because it can mask the no value set case.

    +
    -
    -ident(tree: Tree) None[source]
    -
    +
    +get(name: str) BoolType | BytesType | DoubleType | DurationType | IntType | ListType | MapType | None | StringType | TimestampType | UintType | Callable[[...], BoolType | BytesType | DoubleType | DurationType | IntType | ListType | MapType | None | StringType | TimestampType | UintType | CELEvalError | Type[BoolType] | Type[BytesType] | Type[DoubleType] | Type[DurationType] | Type[IntType] | Type[ListType] | Type[MapType] | Callable[[...], None] | Type[StringType] | Type[TimestampType] | Type[TypeType] | Type[UintType] | Type[PackageType] | Type[MessageType]] | NameContainer
    +

    Handle activation.name in transpiled code (or activation.get('name')).

    +

    If the name is not in the Activation with a value, a NameError exception must be raised.

    +

    Note that Activation.resolve_variable() depends on NameContainer.find_name(). +The NameContainer.find_name() method also find the value.

    +

    This is – perhaps – less than optimal because it can mask the no value set case.

    +
    -
    -classmethod in_tree(tree: Tree) str | None[source]
    -
    - -
    -
    -__parameters__ = ()
    -
    +
    +__repr__() str[source]
    +

    Return repr(self).

    +
    @@ -2316,42 +3732,40 @@

    Timzone Details
    -class celpy.evaluation.Evaluator(ast: Tree, activation: Activation, functions: Sequence[Callable[[...], BoolType | BytesType | DoubleType | DurationType | IntType | ListType | MapType | None | StringType | TimestampType | UintType | CELEvalError | Type[BoolType] | Type[BytesType] | Type[DoubleType] | Type[DurationType] | Type[IntType] | Type[ListType] | Type[MapType] | Callable[[...], None] | Type[StringType] | Type[TimestampType] | Type[TypeType] | Type[UintType] | Type[PackageType] | Type[MessageType]]] | Mapping[str, Callable[[...], BoolType | BytesType | DoubleType | DurationType | IntType | ListType | MapType | None | StringType | TimestampType | UintType | CELEvalError | Type[BoolType] | Type[BytesType] | Type[DoubleType] | Type[DurationType] | Type[IntType] | Type[ListType] | Type[MapType] | Callable[[...], None] | Type[StringType] | Type[TimestampType] | Type[TypeType] | Type[UintType] | Type[PackageType] | Type[MessageType]]] | None = None)[source]
    +class celpy.evaluation.Evaluator(ast: Tree, activation: Activation)[source]

    Evaluate an AST in the context of a specific Activation.

    See https://github.com/google/cel-go/blob/master/examples/README.md

    General Evaluation.

    An AST node must call self.visit_children(tree) explicitly to build the values for all the children of this node.

    -

    Exceptions.

    +

    Exceptions

    To handle 2 / 0 || true, the ||, &&, and ?: operators do not trivially evaluate and raise exceptions. They bottle up the exceptions and treat them as a kind of undecided value.

    -

    Identifiers.

    +

    Identifiers

    Identifiers have three meanings:

    • An object. This is either a variable provided in the activation or a function provided when building an execution. Objects also have type annotations.

    • -
    • A type annotation without an object, This is used to build protobuf messages.

    • +
    • A type annotation without an object. This is used to build protobuf messages.

    • A macro name. The member_dot_arg construct may have a macro. Plus the ident_arg construct may also have a dyn() or has() macro. See below for more.

    -

    Other than macros, a name maps to an Referent instance. This will have an -annotation and – perhaps – an associated object.

    +

    Other than macros, a name maps to an Referent instance. +This will have an annotation and – perhaps – an associated object.

    Names have nested paths. a.b.c is a mapping, a, that contains a mapping, b, that contains c.

    -

    MACROS ARE SPECIAL.

    -

    The macros do not all simply visit their children to perform evaluation. -There are three cases:

    +

    There are three cases:

    @@ -2386,18 +3800,24 @@

    Timzone Details
    sub_evaluator(ast: Tree) Evaluator[source]
    -

    Build an evaluator for a sub-expression in a macro. -:param ast: The AST for the expression in the macro. -:return: A new Evaluator instance.

    +

    Build an evaluator for a sub-expression in a macro.

    +
    +
    Parameters:
    +

    ast – The AST for the expression in the macro.

    +
    +
    Returns:
    +

    A new Evaluator instance.

    +
    +
    -set_activation(values: Mapping[str, BoolType | BytesType | DoubleType | DurationType | IntType | ListType | MapType | None | StringType | TimestampType | UintType | CELEvalError | Type[BoolType] | Type[BytesType] | Type[DoubleType] | Type[DurationType] | Type[IntType] | Type[ListType] | Type[MapType] | Callable[[...], None] | Type[StringType] | Type[TimestampType] | Type[TypeType] | Type[UintType] | Type[PackageType] | Type[MessageType] | NameContainer]) Evaluator[source]
    -

    Chain a new activation using the given Context. +set_activation(values: Mapping[str, BoolType | BytesType | DoubleType | DurationType | IntType | ListType | MapType | None | StringType | TimestampType | UintType | CELEvalError | Type[BoolType] | Type[BytesType] | Type[DoubleType] | Type[DurationType] | Type[IntType] | Type[ListType] | Type[MapType] | Callable[[...], None] | Type[StringType] | Type[TimestampType] | Type[TypeType] | Type[UintType] | Type[PackageType] | Type[MessageType] | NameContainer | CELFunction]) Evaluator[source] +

    Create a new activation using the given Context. This is used for two things:

      -
    1. Bind external variables like command-line arguments or environment variables.

    2. +
    3. Bind external variables. Examples are command-line arguments and environment variables.

    4. Build local variable(s) for macro evaluation.

    @@ -2409,17 +3829,18 @@

    Timzone Details
    -evaluate() BoolType | BytesType | DoubleType | DurationType | IntType | ListType | MapType | None | StringType | TimestampType | UintType[source]
    +evaluate(context: Mapping[str, BoolType | BytesType | DoubleType | DurationType | IntType | ListType | MapType | None | StringType | TimestampType | UintType | CELEvalError | Type[BoolType] | Type[BytesType] | Type[DoubleType] | Type[DurationType] | Type[IntType] | Type[ListType] | Type[MapType] | Callable[[...], None] | Type[StringType] | Type[TimestampType] | Type[TypeType] | Type[UintType] | Type[PackageType] | Type[MessageType] | NameContainer | CELFunction] | None = None) BoolType | BytesType | DoubleType | DurationType | IntType | ListType | MapType | None | StringType | TimestampType | UintType[source]

    Evaluate this AST and return the value or raise an exception.

    There are two variant use cases.

    • External clients want the value or the exception.

    • -
    • Internally, we sometimes want to silence CELEvalError exceptions so that +

    • Internally, we sometimes want to silence CELEvalError exceptions so that we can apply short-circuit logic and choose a non-exceptional result.

    @@ -2436,7 +3857,7 @@

    Timzone Details
  • Object creation and type conversions.

  • -
  • Other built-in functions like size()

  • +
  • Other functions like size() or type()

  • Extension functions

  • @@ -2444,8 +3865,10 @@

    Timzone Details
    method_eval(object: BoolType | BytesType | DoubleType | DurationType | IntType | ListType | MapType | None | StringType | TimestampType | UintType | CELEvalError | Type[BoolType] | Type[BytesType] | Type[DoubleType] | Type[DurationType] | Type[IntType] | Type[ListType] | Type[MapType] | Callable[[...], None] | Type[StringType] | Type[TimestampType] | Type[TypeType] | Type[UintType] | Type[PackageType] | Type[MessageType], method_ident: Token, exprlist: Iterable[BoolType | BytesType | DoubleType | DurationType | IntType | ListType | MapType | None | StringType | TimestampType | UintType | CELEvalError | Type[BoolType] | Type[BytesType] | Type[DoubleType] | Type[DurationType] | Type[IntType] | Type[ListType] | Type[MapType] | Callable[[...], None] | Type[StringType] | Type[TimestampType] | Type[TypeType] | Type[UintType] | Type[PackageType] | Type[MessageType]] | None = None) BoolType | BytesType | DoubleType | DurationType | IntType | ListType | MapType | None | StringType | TimestampType | UintType | CELEvalError | Type[BoolType] | Type[BytesType] | Type[DoubleType] | Type[DurationType] | Type[IntType] | Type[ListType] | Type[MapType] | Callable[[...], None] | Type[StringType] | Type[TimestampType] | Type[TypeType] | Type[UintType] | Type[PackageType] | Type[MessageType][source]
    -

    Method evaluation. While are (nominally) attached to an object, the only thing -actually special is that the object is the first parameter to a function.

    +

    Method evaluation. While these are (nominally) attached to an object, +that would make overrides complicated. +Instead, these are functions (which can be overridden). +The object must the first parameter to a function.

    @@ -2462,14 +3885,14 @@

    Timzone Details
  • If f is a repeated field or map field, has(e.f) indicates whether the field is non-empty.

  • -
  • If f is a singular or oneof field, has(e.f) indicates whether the field is set.

  • +
  • If f is a singular or “oneof” field, has(e.f) indicates whether the field is set.

  • If e evaluates to a protocol buffers version 3 message and f is a defined field:

  • The AST doesn’t provide a flat list of values, however.

    @@ -2546,7 +3969,7 @@

    Timzone Details
    values = self.visit_children(tree)
     func = functions[op_name_map[tree.data]]
    -result = func(*values)
    +result_value = func(*values)
     

    The AST doesn’t provide a flat list of values, however.

    @@ -2563,7 +3986,7 @@

    Timzone Details
    values = self.visit_children(tree)
     func = functions[op_name_map[tree.data]]
    -result = func(*values)
    +result_value = func(*values)
     

    The AST doesn’t provide a flat list of values, however.

    @@ -2579,7 +4002,7 @@

    Timzone Details
    values = self.visit_children(tree)
     func = functions[op_name_map[tree.data]]
    -result = func(*values)
    +result_value = func(*values)
     

    But, values has the structure [[], right]

    @@ -2619,7 +4042,7 @@

    Timzone Details
    build_reduce_macro_eval(child: Tree) Tuple[Callable[[BoolType | BytesType | DoubleType | DurationType | IntType | ListType | MapType | None | StringType | TimestampType | UintType | CELEvalError | Type[BoolType] | Type[BytesType] | Type[DoubleType] | Type[DurationType] | Type[IntType] | Type[ListType] | Type[MapType] | Callable[[...], None] | Type[StringType] | Type[TimestampType] | Type[TypeType] | Type[UintType] | Type[PackageType] | Type[MessageType], BoolType | BytesType | DoubleType | DurationType | IntType | ListType | MapType | None | StringType | TimestampType | UintType | CELEvalError | Type[BoolType] | Type[BytesType] | Type[DoubleType] | Type[DurationType] | Type[IntType] | Type[ListType] | Type[MapType] | Callable[[...], None] | Type[StringType] | Type[TimestampType] | Type[TypeType] | Type[UintType] | Type[PackageType] | Type[MessageType]], BoolType | BytesType | DoubleType | DurationType | IntType | ListType | MapType | None | StringType | TimestampType | UintType | CELEvalError | Type[BoolType] | Type[BytesType] | Type[DoubleType] | Type[DurationType] | Type[IntType] | Type[ListType] | Type[MapType] | Callable[[...], None] | Type[StringType] | Type[TimestampType] | Type[TypeType] | Type[UintType] | Type[PackageType] | Type[MessageType]], Tree][source]
    -

    Builds macro function and intiial expression for reduce().

    +

    Builds macro function and initial expression for reduce().

    For example

    [0, 1, 2].reduce(r, i, 0, r + 2*i+1)

    @@ -2680,7 +4103,7 @@

    Timzone Details @@ -2712,16 +4135,6 @@

    Timzone Details -
    -__abstractmethods__ = frozenset({})
    -
    - -
    -
    -__parameters__ = ()
    -
    -
    member_object(tree: Tree) BoolType | BytesType | DoubleType | DurationType | IntType | ListType | MapType | None | StringType | TimestampType | UintType | CELEvalError | Type[BoolType] | Type[BytesType] | Type[DoubleType] | Type[DurationType] | Type[IntType] | Type[ListType] | Type[MapType] | Callable[[...], None] | Type[StringType] | Type[TimestampType] | Type[TypeType] | Type[UintType] | Type[PackageType] | Type[MessageType][source]
    @@ -2750,11 +4163,7 @@

    Timzone Detailsprimary() is similar to method(). -Each of the individual rules then works with a tree instead of a child of the -primary tree.

    -

    This includes function-like macros: has() and dyn(). +

    This includes function-like macros: has() and dyn(). These are special cases and cannot be overridden.

    @@ -2786,328 +4195,546 @@

    Timzone Details{'a': 1, 'b': 2/0}['a'] a meaningful result in CEL? -Or is this an error because the entire member is erroneous?

    +
    +
    +__abstractmethods__ = frozenset({})
    +
    + +
    +
    +__parameters__ = ()
    +
    + +
    +
    +celpy.evaluation.the_activation: Activation
    +
    +
    -
    -celpy.evaluation.celstr(token: Token) StringType[source]
    -

    Evaluate a CEL string literal, expanding escapes to create a Python string.

    -

    It may be that built-in eval() might work for some of this, but -the octal escapes aren’t really viable.

    -
    -
    Parameters:
    -

    token – CEL token value

    -
    -
    Returns:
    -

    str

    -
    -
    +
    +celpy.evaluation.result(activation: Activation, cel_expr: Callable[[Activation], BoolType | BytesType | DoubleType | DurationType | IntType | ListType | MapType | None | StringType | TimestampType | UintType | CELEvalError | Type[BoolType] | Type[BytesType] | Type[DoubleType] | Type[DurationType] | Type[IntType] | Type[ListType] | Type[MapType] | Callable[[...], None] | Type[StringType] | Type[TimestampType] | Type[TypeType] | Type[UintType] | Type[PackageType] | Type[MessageType]]) BoolType | BytesType | DoubleType | DurationType | IntType | ListType | MapType | None | StringType | TimestampType | UintType | CELEvalError | Type[BoolType] | Type[BytesType] | Type[DoubleType] | Type[DurationType] | Type[IntType] | Type[ListType] | Type[MapType] | Callable[[...], None] | Type[StringType] | Type[TimestampType] | Type[TypeType] | Type[UintType] | Type[PackageType] | Type[MessageType][source]
    +

    Implements “checked exception” handling for CEL expressions transpiled to Python. +An expression must be wrapped by a lambda. +The lambda is evaluated by this function; a subset of Python exceptions become CELEvalError objects.

    +
    >>> some_activation = Activation()
    +
    +
    +

    Within the CEL transpiled code, we can now use code like this…

    +
    >>> expr = lambda activation: 355 / 0
    +>>> result(some_activation, expr)
    +CELEvalError(*('divide by zero', <class 'ZeroDivisionError'>, ('division by zero',)))
    +
    +
    +

    The exception becomes an object.

    -
    -celpy.evaluation.celbytes(token: Token) BytesType[source]
    -

    Evaluate a CEL bytes literal, expanding escapes to create a Python bytes object.

    -
    -
    Parameters:
    -

    token – CEL token value

    -
    -
    Returns:
    -

    bytes

    -
    -
    +
    +celpy.evaluation.macro_map(activation: Activation, bind_variable: str, cel_expr: Callable[[Activation], BoolType | BytesType | DoubleType | DurationType | IntType | ListType | MapType | None | StringType | TimestampType | UintType], cel_gen: Callable[[Activation], Iterable[Activation]]) BoolType | BytesType | DoubleType | DurationType | IntType | ListType | MapType | None | StringType | TimestampType | UintType | CELEvalError | Type[BoolType] | Type[BytesType] | Type[DoubleType] | Type[DurationType] | Type[IntType] | Type[ListType] | Type[MapType] | Callable[[...], None] | Type[StringType] | Type[TimestampType] | Type[TypeType] | Type[UintType] | Type[PackageType] | Type[MessageType][source]
    +

    The results of a source.map(v, expr) macro: a list of values.

    - -
    -

    parser

    -

    CEL Parser.

    -

    See https://github.com/google/cel-spec/blob/master/doc/langdef.md

    -

    https://github.com/google/cel-cpp/blob/master/parser/Cel.g4

    -

    https://github.com/google/cel-go/blob/master/parser/gen/CEL.g4

    -

    Builds a parser from the supplied cel.lark grammar.

    -

    Example:

    -
    >>> from celpy.celparser import CELParser
    ->>> p = CELParser()
    ->>> text2 = 'type(null)'
    ->>> ast2 = p.parse(text2)
    ->>> print(ast2.pretty().replace("   ","   "))
    -expr
    -  conditionalor
    -    conditionaland
    -      relation
    -        addition
    -          multiplication
    -            unary
    -              member
    -                primary
    -                  ident_arg
    -                    type
    -                    exprlist
    -                      expr
    -                        conditionalor
    -                          conditionaland
    -                            relation
    -                              addition
    -                                multiplication
    -                                  unary
    -                                    member
    -                                      primary
    -                                        literal    null
    -
    -
    -
    -
    -exception celpy.celparser.CELParseError(*args: Any, line: int | None = None, column: int | None = None)[source]
    -
    -
    -__init__(*args: Any, line: int | None = None, column: int | None = None) None[source]
    -
    +
    +
    +celpy.evaluation.macro_filter(activation: Activation, bind_variable: str, cel_expr: Callable[[Activation], BoolType | BytesType | DoubleType | DurationType | IntType | ListType | MapType | None | StringType | TimestampType | UintType], cel_gen: Callable[[Activation], Iterable[Activation]]) BoolType | BytesType | DoubleType | DurationType | IntType | ListType | MapType | None | StringType | TimestampType | UintType | CELEvalError | Type[BoolType] | Type[BytesType] | Type[DoubleType] | Type[DurationType] | Type[IntType] | Type[ListType] | Type[MapType] | Callable[[...], None] | Type[StringType] | Type[TimestampType] | Type[TypeType] | Type[UintType] | Type[PackageType] | Type[MessageType][source]
    +

    The results of a source.filter(v, expr) macro: a list of values.

    +
    + +
    +
    +celpy.evaluation.macro_exists_one(activation: Activation, bind_variable: str, cel_expr: Callable[[Activation], BoolType | BytesType | DoubleType | DurationType | IntType | ListType | MapType | None | StringType | TimestampType | UintType], cel_gen: Callable[[Activation], Iterable[Activation]]) BoolType | BytesType | DoubleType | DurationType | IntType | ListType | MapType | None | StringType | TimestampType | UintType | CELEvalError | Type[BoolType] | Type[BytesType] | Type[DoubleType] | Type[DurationType] | Type[IntType] | Type[ListType] | Type[MapType] | Callable[[...], None] | Type[StringType] | Type[TimestampType] | Type[TypeType] | Type[UintType] | Type[PackageType] | Type[MessageType][source]
    +

    The results of a source.exists_one(v, expr) macro: a list of values.

    +

    Note the short-circuit concept. +Count the True; Break on an Exception

    +
    +
    +
    +celpy.evaluation.macro_exists(activation: Activation, bind_variable: str, cel_expr: Callable[[Activation], BoolType | BytesType | DoubleType | DurationType | IntType | ListType | MapType | None | StringType | TimestampType | UintType], cel_gen: Callable[[Activation], Iterable[Activation]]) BoolType | BytesType | DoubleType | DurationType | IntType | ListType | MapType | None | StringType | TimestampType | UintType | CELEvalError | Type[BoolType] | Type[BytesType] | Type[DoubleType] | Type[DurationType] | Type[IntType] | Type[ListType] | Type[MapType] | Callable[[...], None] | Type[StringType] | Type[TimestampType] | Type[TypeType] | Type[UintType] | Type[PackageType] | Type[MessageType][source]
    +

    The results of a source.exists(v, expr) macro: a list of values.

    +
    + +
    +
    +celpy.evaluation.macro_all(activation: Activation, bind_variable: str, cel_expr: Callable[[Activation], BoolType | BytesType | DoubleType | DurationType | IntType | ListType | MapType | None | StringType | TimestampType | UintType], cel_gen: Callable[[Activation], Iterable[Activation]]) BoolType | BytesType | DoubleType | DurationType | IntType | ListType | MapType | None | StringType | TimestampType | UintType | CELEvalError | Type[BoolType] | Type[BytesType] | Type[DoubleType] | Type[DurationType] | Type[IntType] | Type[ListType] | Type[MapType] | Callable[[...], None] | Type[StringType] | Type[TimestampType] | Type[TypeType] | Type[UintType] | Type[PackageType] | Type[MessageType][source]
    +

    The results of a source.all(v, expr) macro: a list of values.

    -
    -class celpy.celparser.CELParser[source]
    -

    Wrapper for the CEL parser and the syntax error messages.

    +
    +class celpy.evaluation.TranspilerTree(data: str, children: Sequence[Token | TranspilerTree], meta: Meta | None = None)[source]
    +
    +
    +data: str
    +
    +
    -
    -CEL_PARSER: Lark | None = None
    +
    +children: Sequence[Token | TranspilerTree]
    -
    -__init__() None[source]
    +
    +__init__(data: str, children: Sequence[Token | TranspilerTree], meta: Meta | None = None) None[source]
    +
    + +
    +
    +__parameters__ = ()
    +
    + +
    + +
    +
    +class celpy.evaluation.Transpiler(ast: TranspilerTree, activation: Activation)[source]
    +

    Transpile the CEL construct(s) to Python functions. +This is a Facade that wraps two visitor subclasses to do two phases +of transpilation.

    +

    The resulting Python code can be used with compile() and exec().

    +
    +
    Phase I:
    +

    The easy transpilation. +It builds simple text expressions for each node of the AST. +This sets aside exception-checking code including short-circuit logic operators and macros. +This decorates the AST with transpiled Python where possible. +It can also decorate with Template objects that require text from children.

    +
    +
    Phase II:
    +

    Collects a sequence of statements. +All of the exception-checking for short-circuit logic operators and macros is packaged as lambdas +that may (or may not) be evaluated.

    +
    +
    +

    Ideally, there could be a Transpiler ABC, and the PythonTranspiler defined as a subclass. +Pragmatically, we can’t see any other sensible transpilation.

    +

    Exception Checking

    +

    To handle 2 / 0 || true, the ||, &&, and ?: operators +the generated code creates lambdas to avoid execution where possible. +An alternative is a Monad-like structure to bottle up an exception, silencing it if it’s unused.

    +

    Identifiers

    +

    Identifiers have three meanings:

    +
      +
    • An object. This is either a variable provided in an Activation or a function provided +when building an execution. Objects also have type annotations.

    • +
    • A type annotation without an object. This can used to build protobuf messages.

    • +
    • A macro name. The member_dot_arg construct (e.g., member.map(v, expr)) may have a macro instead of a method. +Plus the ident_arg construct may be a dyn() or has() macro instead of a function

    • +
    +

    Other than macros, a name maps to an Referent instance. +This will have an annotation and – perhaps – an associated object.

    +

    There are two functions that are macro-like:

    +
      +
    • dyn() does effectively nothing. +It visits its children, but also provides progressive type resolution +through detailed type annotation of the AST.

    • +
    • has() attempts to visit the child and does a boolean transformation +on the resulting exception or value. +This is a macro because it doesn’t raise the exception for a missing +member item reference, but instead maps any exception to False. +It doesn’t return the value found for a member item reference; instead, it maps +successfully finding a member to True.

    • +
    +

    The member and expression list of a macro are transformed into lambdas for use by +special macro_{name} functions. These functions provided the necessary generator +expression to provide CEL semantics.

    +

    Names have nested paths. For example, a.b.c is a mapping a, that contains a mapping, b, +that contains a name c.

    +

    The member() method implements the macro evaluation behavior. +It does not always trivially descend into the children. +In the case of macros, the member evaluates one child tree in the presence +of values from another child tree using specific variable binding in a kind +of stack frame.

    +
    +
    +logger = <Logger celpy.Transpiler (WARNING)>
    -
    -static ambiguous_literals(t: Token) Token[source]
    -

    Resolve a grammar ambiguity between identifiers and literals

    +
    +__init__(ast: TranspilerTree, activation: Activation) None[source]
    +

    Create the Transpiler for an AST with specific variables and functions.

    +
    +
    Parameters:
    +
      +
    • ast – The AST to transpile.

    • +
    • activation – An activation with functions and types to use.

    • +
    +
    +
    -
    -parse(text: str) Tree[source]
    -
    +
    +transpile() None[source]
    +

    Two-phase transpilation.

    +
      +
    1. Decorate AST with the most constructs.

    2. +
    3. Expand into statements for lambdas that wrap checked exceptions.

    4. +
    +
    -
    -error_text(message: str, line: int | None = None, column: int | None = None) str[source]
    +
    +evaluate(context: Mapping[str, BoolType | BytesType | DoubleType | DurationType | IntType | ListType | MapType | None | StringType | TimestampType | UintType | CELEvalError | Type[BoolType] | Type[BytesType] | Type[DoubleType] | Type[DurationType] | Type[IntType] | Type[ListType] | Type[MapType] | Callable[[...], None] | Type[StringType] | Type[TimestampType] | Type[TypeType] | Type[UintType] | Type[PackageType] | Type[MessageType] | NameContainer | CELFunction]) BoolType | BytesType | DoubleType | DurationType | IntType | ListType | MapType | None | StringType | TimestampType | UintType[source]
    -
    -class celpy.celparser.DumpAST[source]
    -

    Dump a CEL AST creating a close approximation to the original source.

    -
    -
    -classmethod display(ast: Tree) str[source]
    -
    - +
    +class celpy.evaluation.Phase1Transpiler(facade: Transpiler)[source]
    +

    Decorate all nodes with transpiled Python code, where possible. +For short-circuit operators or macros, where a “checked exception” is required, +a simple ex_{n} name is present, and separate statements are provided as +a decoration to handle the more complicated cases.

    +

    Each construct has an associated Template. +For the simple cases, the transpiled value is the entire expression.

    +
    >>> from unittest.mock import Mock
    +>>> source = "7 * (3 + 3)"
    +>>> parser = celpy.CELParser()
    +>>> tree = parser.parse(source)
    +>>> tp = Phase1Transpiler(Mock(base_activation=celpy.Activation()))
    +>>> _ = tp.visit(tree)
    +>>> tree.transpiled
    +'operator.mul(celpy.celtypes.IntType(7), operator.add(celpy.celtypes.IntType(3), celpy.celtypes.IntType(3)))'
    +
    +
    +

    Some constructs wrap macros or short-circuit logic, and require a more sophisticated execution. +There will be “checked exceptions”, returned as values. +This requires statements with lambdas that can be wrapped by the result() function.

    +

    The Phase2Transpiler does this transformation from expressions to a sequence of statements.

    -
    -__init__() None[source]
    +
    +__init__(facade: Transpiler) None[source]
    -
    -expr(tree: Tree) None[source]
    -
    +
    +visit(tree: TranspilerTree) TranspilerTree[source]
    +

    Initialize the decorations for each node.

    +
    -
    -conditionalor(tree: Tree) None[source]
    -
    +
    +func_name(label: str) str[source]
    +

    Provide a transpiler-friendly name for the function.

    +

    Some internally-defined functions appear to come from _operator module. +We need to rename some celpy functions to be from operator.

    +

    Some functions – specifically lt, le, gt, ge, eq, ne – are wrapped boolean(operator.f) +obscuring their name.

    +
    -
    -conditionaland(tree: Tree) None[source]
    -
    +
    +expr(tree: TranspilerTree) None[source]
    +

    expr : conditionalor [“?” conditionalor “:” expr]

    +
    -
    -relation(tree: Tree) None[source]
    -
    +
    +conditionalor(tree: TranspilerTree) None[source]
    +

    conditionalor : [conditionalor “||”] conditionaland

    +
    -
    -relation_lt(tree: Tree) None[source]
    -
    +
    +conditionaland(tree: TranspilerTree) None[source]
    +

    conditionaland : [conditionaland “&&”] relation

    +
    -
    -relation_le(tree: Tree) None[source]
    -
    +
    +relation(tree: TranspilerTree) None[source]
    +
    +
    relation[relation_lt | relation_le | relation_ge | relation_gt
    +
    relation_eq | relation_ne | relation_in] addition
    +
    +
    +
    +

    relation_lt : relation “<” +relation_le : relation “<=” +relation_gt : relation “>” +relation_ge : relation “>=” +relation_eq : relation “==” +relation_ne : relation “!=” +relation_in : relation “in”

    +
    -
    -relation_gt(tree: Tree) None[source]
    -
    +
    +addition(tree: TranspilerTree) None[source]
    +

    addition : [addition_add | addition_sub] multiplication

    +

    addition_add : addition “+” +addition_sub : addition “-”

    +
    -
    -relation_ge(tree: Tree) None[source]
    -
    +
    +multiplication(tree: TranspilerTree) None[source]
    +

    multiplication : [multiplication_mul | multiplication_div | multiplication_mod] unary

    +

    multiplication_mul : multiplication “*” +multiplication_div : multiplication “/” +multiplication_mod : multiplication “%”

    +
    -
    -relation_eq(tree: Tree) None[source]
    -
    +
    +unary(tree: TranspilerTree) None[source]
    +

    unary : [unary_not | unary_neg] member

    +

    unary_not : “!” +unary_neg : “-”

    +
    -
    -relation_ne(tree: Tree) None[source]
    -
    +
    +member(tree: TranspilerTree) None[source]
    +

    member : member_dot | member_dot_arg | member_item | member_object | primary

    +
    -
    -relation_in(tree: Tree) None[source]
    -
    +
    +member_dot(tree: TranspilerTree) None[source]
    +

    member_dot : member “.” IDENT

    +
    +

    Important

    +

    The member can be any of a variety of objects:

    +
      +
    • NameContainer(Dict[str, Referent])

    • +
    • Activation

    • +
    • MapType(Dict[Value, Value])

    • +
    • MessageType(MapType)

    • +
    +

    All of which define a get() method. +The nuance is the NameContainer is also a Python dict and there’s an +overload issue between that get() and other get() definitions.

    +
    +
    -
    -addition(tree: Tree) None[source]
    -
    +
    +member_dot_arg(tree: TranspilerTree) None[source]
    +

    member_dot_arg : member “.” IDENT “(” [exprlist] “)”

    +

    Two flavors: macro and non-macro.

    +
    -
    -addition_add(tree: Tree) None[source]
    -
    +
    +member_index(tree: TranspilerTree) None[source]
    +

    member_item : member “[” expr “]”

    +
    -
    -addition_sub(tree: Tree) None[source]
    -
    +
    +member_object(tree: TranspilerTree) None[source]
    +

    member_object : member “{” [fieldinits] “}”

    +
    -
    -multiplication(tree: Tree) None[source]
    -
    +
    +primary(tree: TranspilerTree) None[source]
    +
    +
    primarydot_ident_arg | dot_ident | ident_arg | ident
    +
    paren_expr | list_lit | map_lit | literal
    +
    +
    +
    +
    -
    -multiplication_mul(tree: Tree) None[source]
    -
    +
    +dot_ident_arg(tree: TranspilerTree) None[source]
    +

    dot_ident_arg : “.” IDENT “(” [exprlist] “)”

    +
    -
    -multiplication_div(tree: Tree) None[source]
    -
    +
    +dot_ident(tree: TranspilerTree) None[source]
    +

    dot_ident : “.” IDENT

    +
    -
    -multiplication_mod(tree: Tree) None[source]
    -
    +
    +ident_arg(tree: TranspilerTree) None[source]
    +

    ident_arg : IDENT “(” [exprlist] “)”

    +
    -
    -unary(tree: Tree) None[source]
    -
    +
    +ident(tree: TranspilerTree) None[source]
    +

    ident : IDENT

    +
    -
    -unary_not(tree: Tree) None[source]
    -
    +
    +paren_expr(tree: TranspilerTree) None[source]
    +

    paren_expr : “(” expr “)”

    +
    -
    -unary_neg(tree: Tree) None[source]
    -
    +
    +list_lit(tree: TranspilerTree) None[source]
    +

    list_lit : “[” [exprlist] “]”

    +
    -
    -member_dot(tree: Tree) None[source]
    -
    +
    +map_lit(tree: TranspilerTree) None[source]
    +

    map_lit : “{” [mapinits] “}”

    +
    -
    -member_dot_arg(tree: Tree) None[source]
    -
    +
    +exprlist(tree: TranspilerTree) None[source]
    +

    exprlist : expr (“,” expr)*

    +
    -
    -member_index(tree: Tree) None[source]
    -
    +
    +fieldinits(tree: TranspilerTree) None[source]
    +

    fieldinits : IDENT “:” expr (“,” IDENT “:” expr)*

    +
    -
    -member_object(tree: Tree) None[source]
    -
    +
    +mapinits(tree: TranspilerTree) None[source]
    +

    mapinits : expr “:” expr (“,” expr “:” expr)*

    +
    -
    -dot_ident_arg(tree: Tree) None[source]
    -
    +
    +literal(tree: TranspilerTree) None[source]
    +
    +
    literalUINT_LIT | FLOAT_LIT | INT_LIT | MLSTRING_LIT | STRING_LIT | BYTES_LIT
    +
    BOOL_LIT | NULL_LIT
    +
    +
    +
    +
    -
    -
    -dot_ident(tree: Tree) None[source]
    +
    +
    +__parameters__ = ()
    -
    -
    -ident_arg(tree: Tree) None[source]
    -
    +
    +
    +
    +class celpy.evaluation.Phase2Transpiler(facade: Transpiler)[source]
    +

    Extract any checked_exception evaluation statements that decorate the parse tree. +Also, get the overall top-level expression, assigned to special variable, CEL.

    +
    >>> from unittest.mock import Mock
    +>>> from pprint import pprint
    +>>> source = '["hello", "world"].map(x, x) == ["hello", "world"]'
    +>>> celpy.CELParser.CEL_PARSER = None
    +>>> parser = celpy.CELParser(tree_class=celpy.evaluation.TranspilerTree)
    +>>> tree = parser.parse(source)
    +>>> tp1 = Phase1Transpiler(Mock(base_activation=celpy.Activation()))
    +>>> _ = tp1.visit(tree)
    +>>> tp2 = Phase2Transpiler(Mock(base_activation=celpy.Activation()))
    +>>> _ = tp2.visit(tree)
    +>>> pprint(tp2.statements(tree), width=256)
    +['# member_dot_arg map:',
    + "ex_10_l = lambda activation: celpy.celtypes.ListType([celpy.celtypes.StringType('hello'), celpy.celtypes.StringType('world')])",
    + 'ex_10_x = lambda activation: activation.x',
    + "ex_10 = lambda activation: celpy.evaluation.macro_map(activation, 'x', ex_10_x, ex_10_l)",
    + "CEL = celpy.evaluation.result(base_activation, lambda activation: celpy.evaluation.bool_eq(ex_10(activation), celpy.celtypes.ListType([celpy.celtypes.StringType('hello'), celpy.celtypes.StringType('world')])))\n"]
    +
    +
    -
    -ident(tree: Tree) None[source]
    +
    +__init__(facade: Transpiler) None[source]
    -
    -
    -paren_expr(tree: Tree) None[source]
    +
    +
    +__parameters__ = ()
    -
    -list_lit(tree: Tree) None[source]
    -
    +
    +expr(tree: TranspilerTree) None[source]
    +

    expr : conditionalor [“?” conditionalor “:” expr]

    +

    All checked_exception structures are a tuple[Template, dict[str, Callable[[Tree], str]]

    +
    -
    -map_lit(tree: Tree) None[source]
    -
    +
    +conditionalor(tree: TranspilerTree) None
    +

    expr : conditionalor [“?” conditionalor “:” expr]

    +

    All checked_exception structures are a tuple[Template, dict[str, Callable[[Tree], str]]

    +
    -
    -exprlist(tree: Tree) None[source]
    -
    +
    +conditionaland(tree: TranspilerTree) None
    +

    expr : conditionalor [“?” conditionalor “:” expr]

    +

    All checked_exception structures are a tuple[Template, dict[str, Callable[[Tree], str]]

    +
    -
    -fieldinits(tree: Tree) None[source]
    -
    +
    +member_dot_arg(tree: TranspilerTree) None
    +

    expr : conditionalor [“?” conditionalor “:” expr]

    +

    All checked_exception structures are a tuple[Template, dict[str, Callable[[Tree], str]]

    +
    -
    -mapinits(tree: Tree) None[source]
    -

    Note reversed pop order for values and keys.

    +
    +ident_arg(tree: TranspilerTree) None
    +

    expr : conditionalor [“?” conditionalor “:” expr]

    +

    All checked_exception structures are a tuple[Template, dict[str, Callable[[Tree], str]]

    -
    -literal(tree: Tree) None[source]
    -
    +
    +statements(tree: TranspilerTree) list[str][source]
    +

    Appends the final CEL = … statement to the sequence of statements, +and returns the transpiled code.

    +

    Two patterns:

    +
      +
    1. Top-most expr was a deferred template, and already is a lambda. +It will be a string of the form "ex_\d+(activation)".

    2. +
    3. Top-most expr was not a deferred template, and needs a lambda wrapper. +It will not be a simple "ex_\d+" reference.

    4. +
    +
    -
    -
    -__parameters__ = ()
    -
    + +
    +
    +celpy.evaluation.celstr(token: Token) StringType[source]
    +

    Evaluate a CEL string literal, expanding escapes to create a Python string.

    +

    It may be that built-in eval() might work for some of this, but +the octal escapes aren’t really viable.

    +
    +
    Parameters:
    +

    token – CEL token value

    +
    +
    Returns:
    +

    str

    +
    +
    -
    -celpy.celparser.tree_dump(ast: Tree) str[source]
    -

    Dumps the AST to approximate the original source

    +
    +celpy.evaluation.celbytes(token: Token) BytesType[source]
    +

    Evaluate a CEL bytes literal, expanding escapes to create a Python bytes object.

    +
    +
    Parameters:
    +

    token – CEL token value

    +
    +
    Returns:
    +

    bytes

    +
    +
    @@ -3139,27 +4766,24 @@

    CEL in Python

    Navigation

    -

    Contents:

    +

    Documentation Content:

    diff --git a/docs/build/html/c7n_functions.html b/docs/build/html/c7n_functions.html index 6c11f80..951ab8b 100644 --- a/docs/build/html/c7n_functions.html +++ b/docs/build/html/c7n_functions.html @@ -14,7 +14,7 @@ - + @@ -34,14 +34,12 @@

    C7N Functions Required

    -

    This survey of C7N filter clauses is based on source code and -on an analysis of working policies. The required functions -are grouped into four clusters, depending on the presence of absence of -the “op” operator clause, and the number of resource types using the feature. +

    This survey of C7N filter clauses is based on source code and on an analysis of working policies. +The required functions are grouped into four clusters, depending on the presence of absence of the “op” operator clause, and the number of resource types using the feature. Within each group, they are ranked in order of popularity.

    For each individual type of filter clause, we provide the following details:

      -
    • The C7N sechema definition.

    • +
    • The C7N schema definition.

    • The resource types where the filter type is used.

    • The variant implementations that are registered (if any.)

    • If used in the working policies,

      @@ -207,7 +205,7 @@

      Design Principles
    • Separate from C7N. The CEL processing is outside C7N, and capable of standing alone. CEL is focused on Protobuf (and JSON) objects. The interface to C7N is via the c7nlib library of functions. These do not depend -on imports from the C7N project, but rely on a CELFilter class offering specific methods. +on imports from the C7N project, but rely on a CELFilter class offering specific methods. Access to C7N objects and their associated methods is limited to the features exposed through the function library and the expected class definition.

    • Resource Representation. CEL is focused on Protobuf (and JSON) objects. @@ -9958,14 +9956,14 @@

      CEL in Python

    Navigation

    -

    Contents:

    +

    Documentation Content:

    diff --git a/docs/build/html/cli.html b/docs/build/html/cli.html index c445516..13e7623 100644 --- a/docs/build/html/cli.html +++ b/docs/build/html/cli.html @@ -14,7 +14,7 @@ - + @@ -35,41 +35,269 @@

    CLI Use of CEL-Python

    -

    We can read JSON directly from stdin, making this a bit like JQ.

    -
    % PYTHONPATH=src python -m celpy '.this.from.json * 3 + 3' <<EOF
    -heredoc> {"this": {"from": {"json": 13}}}
    -heredoc> EOF
    -42
    +

    While CEL-Python’s primary use case is integration into an DSL-based application to provide expressions with a uniform syntax and well-defined semantics. +The expression processing capability is also available as a CLI implemented in the celpy package.

    +
    +

    SYNOPSIS

    +
    python -m celpy [-a name:type=value ...] [-bns] [-p][-d] expr
    +python -m celpy [-a name:type=value ...] -i
     
    +
    +
    +-a <name:type=value>, --arg  <name:type=value>
    +

    Define argument variables, types, and (optional) values. +If the argument value is omitted, then an environment variable will be examined to find the value. +For example, --arg HOME:string makes the HOME environment variable’s value available to the CEL expression.

    +
    + +
    +
    +-b, --boolean
    +

    Return a status code value based on the boolean output.

    +

    true has a status code of 0

    +

    false has a statis code of 1

    +

    Any exception has a stats code of 2

    +
    + +
    +
    +-n, --null-input
    +

    Do not read JSON input from stdin

    +
    + +
    +
    +-s, --slurp
    +

    Treat all input as a single JSON document. +The default is to treat each line of input as a separate NLJSON document.

    +
    + +
    +
    +-i, --interactive
    +

    Operate interactively from a CEL> prompt. +In -i mode, the rest of the options are ignored.

    +
    + +
    +
    +-p, --json-package
    +

    Each NDJSON input (or the single input in -s mode) +is a CEL package.

    +
    + +
    +
    +-d, --json-document
    +

    Each NDJSON input (or the single input in -s mode) +is a separate CEL variable.

    +
    + +
    +
    +-f <spec>, --format <spec>
    +

    Use Python formating instead of JSON conversion of results; +Example --format .6f to format a DoubleType result

    +
    + +
    +
    +expr
    +

    A CEL expression to evaluate.

    +
    + +
    +
    +

    DESCRIPTION

    +

    This provides shell-friendly expression processing. +It follows patterns from several programs.

    +
    +
    jq:
    +

    The celpy application will read newline-delimited JSON +from stdin. +It can also read a single, multiline JSON document in --slurp mode.

    +

    This will evaluate the expression for each JSON document.

    +
    +

    Note

    +

    jq uses . to refer the current document. By setting a package +name of "jq" with the -p option, e.g., -p jq, +and placing the JSON object in the same package, we achieve +similar syntax.

    +
    +
    +
    expr:
    +

    The celpy application does everything expr does, but the syntax is different.

    +

    The output of comparisons in celpy is boolean, where by default. +The expr program returns an integer 1 or 0. +Use the -f option, for example, -f 'd' to see decimal output instead of Boolean text values.

    +
    +
    test:
    +

    This does what test does using CEL syntax. +The stat() function retrieves a mapping with various file status values.

    +

    Use the -b option to set the exit status code from the Boolean result.

    +

    A true value becomes a 0 exit code.

    +

    A false value becomes a 1 exit code.

    +
    +
    bc:
    +

    THe little-used linux bc application has several complex function definitions and other programming support. +CEL can evaluate some bc\ -like expressions. +It could be extended to mimic bc.

    +
    +
    +

    Additionally, in --interactive mode, +there’s a REPL with a CEL> prompt.

    +
    +

    Arguments, Types, and Namespaces

    +

    The --arg options must provide a variable name and type. +CEL objects rely on the celpy.celtypes definitions.

    +

    Because of the close association between CEL and protobuf, some well-known protobuf types +are also supported.

    +

    The value for a variable is optional. +If it is not provided, then the variable is presumed to be an environment variable. +While many environment variables are strings, the type is still required. +For example, use --arg HOME:string to get the value of the HOME environment variable.

    +
    +
    +
    +

    FILES

    +

    By default, JSON documents are read from stdin in NDJSON format (http://jsonlines.org/, http://ndjson.org/). +For each JSON document, the expression is evaluated with the document in a default +package. This allows .name to pick items from the document.

    +

    By default, the output is JSON serialized. +This means strings will be JSON-ified and have quotes. +Using the -f option will expect a single, primitive type that can be formatting using Python’s string formatting mini-language.

    +
    +
    +

    ENVIRONMENT VARIABLES

    +

    Enhanced logging is available when CEL_TRACE is defined. +This is quite voluminous; tracing most pieces of the AST during evaluation.

    +
    +
    +

    CONFIGURATION

    +

    Logging configuration is read from the celpy.toml file. +See Configuration for details.

    +
    +
    +

    EXIT STATUS

    +

    Normally, it’s zero.

    +

    When the -b option is used then the final expression determines the status code.

    +

    A value of true returns 0.

    +

    A value of false returns 1.

    +

    Other values or an evaluation error exception will return 2.

    +
    +
    +

    EXAMPLES

    +

    We can read JSON directly from stdin, making this a bit like the jq application. +We provide a JQ expression, '.this.from.json * 3 + 3', and a JSON document. +The standard output is the computed result.

    +
    % python -m celpy '.this.from.json * 3 + 3' <<EOF
    +heredoc> {"this": {"from": {"json": 13}}}
    +heredoc> EOF
    +42
    +
    +
    +

    The default behavior is to read and process stdin, where each line is a separate JSON document. +This is the Newline-Delimited JSON format. +(See https://jsonlines.org and https://github.com/ndjson/ndjson-spec).

    +

    The -s/--slurp treats the stdin as a single JSON document, spread over multiple lines. +This parallels the way the the jq application handles JSON input.

    +

    We can avoid reading stdin by using the -n/--null-input option. +This option will evaluate the expression using only command-line argument values.

    It’s also a desk calculator.

    -
    % python -m celpy -n '355.0 / 113.0'
    -3.1415929203539825
    +
    % python -m celpy -n '355.0 / 113.0'
    +3.1415929203539825
    +
    +
    +

    And, yes, this use case has a tiny advantage over python -c '355/113'. +Most notably, the ability to embed Google CEL into other contexts where you don’t really want Python’s power. +There’s no CEL import or built-in eval() function to raise security concerns.

    +

    We can provide a -a/--arg option to define a name in the current activation with particular data type. +The expression, 'x * 3 + 3' depends on a x variable, set by the -a option. +Note the variable:type syntax for setting the type of the variable.

    +
    % python -m celpy -n -ax:int=13 'x * 3 + 3'
    +42
    +
    +
    +

    This is what the bash expr command does. +CEL can do more. +For example, floating-point math. +Here we’ve set two variables, x and tot, before evaluating an expression.

    +
    % python -m celpy -n -ax:double=113 -atot:double=355 '100. * x/tot'
    +31.830985915492956
     
    -

    And, yes, this has a tiny advantage over python -c '355/113'. Most notably, the ability -to embed Google CEL into other contexts where you don’t really want Python’s power. -There’s no CEL import or built-in exec() function to raise concerns.

    -

    We can provide a -d option to define objects with particular data types, like JSON. -This is particularly helpful for providing protobuf message definitions.

    -
    % PYTHONPATH=src python -m celpy -n -ax:int=13 'x * 3 + 3'
    -42
    +

    If you omit the = from the -a option, then an environment variable’s +value will be bound to the variable name in the activation.

    +
    % TOTAL=41 python -m celpy -n -aTOTAL:int 'TOTAL + 1'
    +42
     
    -

    This command sets a variable x then evaluates the expression. And yes, this is what -expr does. CEL can do more. For example, floating-point math.

    -
    % PYTHONPATH=src python -m celpy -n -ax:double=113 -atot:double=355 '100. * x/tot'
    -31.830985915492956
    +

    Since these operations involves explict type conversions, be aware of the possibility of syntax error exceptions.

    +
    % TOTAL="not a number" python -m celpy -n -aTOTAL:int 'TOTAL + 1'
    +usage: celpy [-h] [-v] [-a ARG] [-n] [-s] [-i] [--json-package NAME] [--json-document NAME] [-b] [-f FORMAT] [expr]
    +celpy: error: argument -a/--arg: arg TOTAL:int value invalid for the supplied type
     
    -

    We can also mimic the test command.

    -
    % PYTHONPATH=src python -m celpy -n -ax:int=113 -atot:int=355 -b 'x > tot'
    -false
    -% echo $?
    -1
    +

    We can also use this instead of the bash test command. +We can bind values with the -a options and then compare them. +The -b/--boolean option sets the status value based on the boolean result value. +The output string is the CEL literal value false. +The status code is a “failure” code of 1.

    +
    % python -m celpy -n -ax:int=113 -atot:int=355 -b 'x > tot'
    +false
    +% echo $?
    +1
     
    -

    The intent is to provide a common implementation for aritmetic and logic.

    +

    Here’s another example that shows the stat() function to get filesystem status.

    +
    % python -m celpy -n -aHOME 'HOME.stat()'
    +{"st_atime": "2025-07-06T20:27:21Z", "st_birthtime": "2006-11-27T18:30:03Z", "st_ctime": "2025-07-06T20:27:20Z", "st_dev": 16777234, "st_ino": 341035, "st_mtime": "2025-07-06T20:27:20Z", "st_nlink": 135, "st_size": 4320, "group_access": true, "user_access": true, "kind": "d", "setuid": false, "setgid": false, "sticky": false, "r": true, "w": true, "x": true, "st_blksize": 4096, "st_blocks": 0, "st_flags": 0, "st_rdev": 0, "st_gen": 0}
    +
    +
    +

    As an example, to compare modification time between two files, use an expression like f1.stat().st_mtime < f2.stat().st_mtime.

    +

    This is longer than the traditional bash expression, but much more clear.

    +

    The file “kind” is a one-letter code: +:b: block +:c: character-mode +:d: directory +:f: regular file +:p: FIFO or pipe +:l: symbolic link +:s: socket

    +

    The r, w, and x attributes indicate if the current effective userid can read, write, or execute the file. This comes from the detailed permission bits.

    +

    The intent is to provide a single, uniform implementation for arithmetic and logic operations. +The primary use case integration into an DSL-based application to provide expressions without the mental burden of writing the parser and evaluator.

    +

    We can also use CEL interactively, because, why not?

    +
    % python -m celpy -i
    +Enter an expression to have it evaluated.
    +CEL> 355. / 113.
    +3.1415929203539825
    +CEL> ?
    +
    +Documented commands (type help <topic>):
    +========================================
    +bye  exit  help  quit  set  show
    +
    +CEL> help set
    +Set variable expression
    +
    +        Evaluates the expression, saves the result as the given variable in the current activation.
    +
    +CEL> set a 6
    +6
    +CEL> set b 7
    +7
    +CEL> a * b
    +42
    +CEL> show
    +{'a': IntType(6), 'b': IntType(7)}
    +CEL> bye
    +%
    +
    +
    +

    The bye, exit, and quit commands all exit the application.

    +
    @@ -98,14 +326,24 @@

    CEL in Python

    Navigation

    -

    Contents:

    +

    Documentation Content:

    @@ -114,7 +352,7 @@

    Related Topics

    diff --git a/docs/build/html/configuration.html b/docs/build/html/configuration.html index b2dcfbd..1594125 100644 --- a/docs/build/html/configuration.html +++ b/docs/build/html/configuration.html @@ -34,20 +34,9 @@
    -

    Configuration

    -

    The CLI application can bind argument values from the environment. -The command-line provides variable names and type information. -The OS environment provides string values.

    -
    export x=6
    -export y=7
    -celpy -n --arg x:int --arg y:int 'x*y'
    -42
    -
    -
    -

    While this example uses the OS environment, -it isn’t the usual sense of configuration. -The only configuration options available for the command-line application are the logging configuration.

    -

    If a celpy.toml file exists in the local directory or the user’s HOME directory, this will be used to provide logging configuration for the celpy application.

    +

    Configuration

    +

    The celpy package uses a configuration file to set the logging options. +If a celpy.toml file exists in the local directory or the user’s HOME directory, this will be used to provide logging configuration for the celpy application.

    This file must have a logging paragraph. This paragraph can contain the parameters for logging configuration.

    [logging]
    @@ -63,12 +52,31 @@ 

    Configuration formatter = "console"

    +

    This provides minimal log output, showing only warnings, errors, and fatal error messages. +The root.level needs to be “INFO” or “DEBUG” to see more output. +Setting a specific logger’s level to “DEBUG” will raise the logging level for a specific component.

    +

    All of the celpy loggers have names starting with celpy.. +This permits integration with other application without polluting those logs with celpy output.

    To enable very detailed debugging, do the following:

    • Set the CEL_TRACE environment variable to some non-empty value, like "true". This enables a @trace decorator on some evaluation methods.

    • -
    • In a [logging.loggers.celpy.Evaluator] paragraph, set level = "DEBUG".

    • -
    • Set the [logging] paragraph, set root.level = "DEBUG".

    • +
    • Add a [logging.loggers.celpy.Evaluator] paragraph, with level = "DEBUG". +This can be done for any of the celpy components with loggers.

    • +
    • In the [logging] paragraph, set root.level = "DEBUG".

    • +
    +

    Loggers include the following:

    +
      +
    • celpy

    • +
    • celpy.Runner

    • +
    • celpy.Environment

    • +
    • celpy.repl

    • +
    • celpy.c7nlib

    • +
    • celpy.celtypes

    • +
    • celpy.evaluation

    • +
    • celpy.NameContainer

    • +
    • celpy.Evaluator

    • +
    • celpy.Transpiler

    @@ -98,14 +106,15 @@

    CEL in Python

    Navigation

    -

    Contents:

    +

    Documentation Content:

    diff --git a/docs/build/html/development.html b/docs/build/html/development.html new file mode 100644 index 0000000..9ab2d03 --- /dev/null +++ b/docs/build/html/development.html @@ -0,0 +1,378 @@ + + + + + + + + Development Tools — CEL in Python documentation + + + + + + + + + + + + + + + + + + + + +
    +
    +
    + + +
    + +
    +

    Development Tools

    +

    The development effort is dependent on several parts of the CEL project.

    +
      +
    1. The language specification. https://github.com/google/cel-spec/blob/master/doc/langdef.md

    2. +
    3. The test cases. https://github.com/google/cel-spec/tree/master/tests/simple/testdata

    4. +
    +

    The language specification is transformed into a Lark grammar. +This is in the src/cel.lark file. +This changes very slowly. +Any changes must be reflected (manually) by revising the lark version of the EBNF.

    +

    The test cases present a more challenging problem.

    +

    A tool, pb2g.py, converts the test cases from Protobuf messages to Gherkin scenarios.

    +

    +@startuml
+
+file source as "source protobuf test cases"
+file features as "Gherkin feature files"
+
+source --> [pb2g.py]
+[pb2g.py] --> [Docker] : "Uses"
+[Docker] ..> [mkgherkin.go] : "Runs"
+[pb2g.py] --> features
+@enduml +

    +
    +

    The pb2g Tool

    +

    The pb2g.py Python application converts a protobuf test case collection into a Gherkin Feature file. +These can be used to update the features directory.

    +
    +

    SYNOPSIS

    +
    +
    +-g <docker | local>, --gherkinizer <docker | local
    +

    Sets the method for converting the Protobuf messages to interim JSON documents. +Either a local Go binary can be run, +or a Docker image, using the tools/Dockerfile can be run. +Using -g docker requires installing Docker (https://www.docker.com)

    +

    Using -g local requires a local version compiled for your platform. +Install the Go tools (https://go.dev/learn/) and compile tools/mkgherkin.go.

    +

    Docker is the default.

    +
    + +
    +
    +-o <output>, --output <output>
    +

    Where to write the feature file. +Generally, it’s helpful to have the .textproto and .feature stems match. +The Makefile assures this.

    +
    + +
    +
    +-s, --silent
    +

    No console output is produced

    +
    + +
    +
    +-v, --verbose
    +

    Verbose debugging output on the console.

    +
    + +
    +
    +source
    +

    A source .textproto file. +This is often the path to a file in a local download of https://github.com/google/cel-spec/tree/master/tests/simple/testdata.

    +

    A URL for the source is not supported.

    +
    + +
    +
    +

    DESCRIPTION

    +

    Convert one .textproto file to a Gherkin .feature file. +There are two steps to the conversion:

    +
      +
    1. Rewrite the .textproto into JSON. +This relies on common Go libraries, and is little more than a syntactic conversion.

    2. +
    3. Rewrite the JSON copy of the .textproto into Gherkin. +This a little more fraught with special cases and exceptions. +The .textproto semantics can be confusing.

    4. +
    +
    +
    +

    FILES

    +
    +
    source:
    +

    A .textproto test case file from the CEL-spec repository.

    +
    +
    output:
    +

    A .feature file with the same stem as the source file is written to the output directory. +basic.textproto will create basic.feature.

    +
    +
    interim:
    +

    An interim JSON-format file is created and deleted. +These are only visible in the event of a fatal error creating the Gherkin output.

    +
    +
    +
    +
    +

    EXAMPLES

    +

    The basic.textproto starts like this:

    +
    name: "basic"
    +description: "Basic conformance tests that all implementations should pass."
    +section {
    +  name: "self_eval_zeroish"
    +  description: "Simple self-evaluating forms to zero-ish values."
    +  test {
    +    name: "self_eval_int_zero"
    +    expr: "0"
    +    value: { int64_value: 0 }
    +  }
    +  test {
    +    name: "self_eval_uint_zero"
    +    expr: "0u"
    +    value: { uint64_value: 0 }
    +  }
    +
    +
    +

    The Feature file created looks like this:

    +
    Feature: basic
    +         Basic conformance tests that all implementations should pass.
    +
    +# self_eval_zeroish -- Simple self-evaluating forms to zero-ish values.
    +
    +Scenario: self_eval_int_zero
    +
    +    When CEL expression "0" is evaluated
    +    #    int64_value:0
    +    Then value is IntType(source=0)
    +
    +
    +Scenario: self_eval_uint_zero
    +
    +    When CEL expression "0u" is evaluated
    +    #    uint64_value:0
    +    Then value is UintType(source=0)
    +
    +
    +

    The source files have a “section” heading which doesn’t have a precise parallel in the Gherkin language. +The sections become comments in the Feature file.

    +

    The features/steps directory has step definition modules that implement the Given, When, and Then clauses.

    +

    Provides step definitions for the c7n_interface.feature. +This is not part of the CEL language specification.

    +

    Provides step definitions for the expr_test_bc.feature, json_query.feature, neither of which are part of the CEL language specificaiton.

    +

    Provides step definitions for the features generated by the pb2g.py tool.

    +
    +
    +
    +

    The features/Makefile

    +

    This Makefile has the following targets:

    +
    +
    %.textproto:
    +

    This copies textproto files from the source directory +to the features directory. +The source is defined by the CEL_SIMPLE_TESTDATA environment variable. +This will overwrite out-of-date files in the features directory.

    +

    It’s important to use git wisely and start with a clean branch of the project so changes can be rolled back.

    +
    +
    %.feature:
    +

    This creates the .feature file from the .textproto file.

    +
    +
    scan:
    +

    This phony target reads all of the .textproto files to be sure they can be converted to Gherkin. +If it concludes with the output "All files scanned successfully", then there are no surprising or unexpected features in the .textproto files.

    +
    +
    clean-broken:
    +

    This phony target removes empty .feature files that may be left over when the conversion process crashes with a fatal error.

    +
    +
    clean-features:
    +

    This phony target removes all of the .textproto-based .feature files. +Manually created .feature files are left intact.

    +
    +
    clean:
    +

    This phony target removes all .textproto and .feature files that are built from the CEL specification. +Manually created .feature files are left intact.

    +
    +
    +

    Currently, the following feature files are built from the CEL specification.

    +
    basic.feature
    +comparisons.feature
    +conversions.feature
    +dynamic.feature
    +enums.feature
    +fields.feature
    +fp_math.feature
    +integer_math.feature
    +lists.feature
    +logic.feature
    +macros.feature
    +namespace.feature
    +parse.feature
    +plumbing.feature
    +proto2.feature
    +proto3.feature
    +string.feature
    +timestamps.feature
    +unknowns.feature
    +
    +
    +
    +
    +

    The docs/Makefile

    +

    This is a Sphinx Makefile to build documentation. +For more information, see https://www.sphinx-doc.org/en/master/index.html

    +
    +
    +

    The Project Makefile

    +

    A top-level Makefile has a number of phony targets:

    +
    +
    build:
    +

    Runs uv build to create a distribution kit.

    +
    +
    install-tools:
    +

    Pulls a golang Docker image and builds the mkgherkin image.

    +
    +
    test:
    +

    Runs the Python 3.12 test environment to execute a quick test.

    +
    +
    test-all:
    +

    Update the features files and run the full test suite.

    +
    +
    test-wip:
    +

    Update the features files and run the WIP test environment – these are tests flagged with @WIP markers.

    +
    +
    test-tools:
    +

    Run a test of only the tools, then scan the features files to be sure they’re still valid after the tool change.

    +
    +
    docs:
    +

    Build the HTML documentation.

    +
    +
    lint:
    +

    Runs the lint test environment to get code coverage, type hint checking, and other lint checks.

    +
    +
    coverage:
    +

    Reproduce the most recent coverage report.

    +
    +
    clean:
    +

    Remove a number of directories and their files:

    +
      +
    • .tox

    • +
    • .Python

    • +
    • bin

    • +
    • include

    • +
    • lib

    • +
    • pip-selfcheck

    • +
    • .json

    • +
    +
    +
    benchmarks:
    +

    Run the applications in the benches directory to gather performance benchmark data.

    +
      +
    • large_resource_set.py

    • +
    • complex_expression.py

    • +
    +
    +
    +
    +
    + + +
    + +
    +
    + +
    +
    + + + + + + + \ No newline at end of file diff --git a/docs/build/html/genindex.html b/docs/build/html/genindex.html index 3dd4f7e..0add273 100644 --- a/docs/build/html/genindex.html +++ b/docs/build/html/genindex.html @@ -34,7 +34,8 @@

    Index

    - _ + Symbols + | _ | A | B | C @@ -42,7 +43,10 @@

    Index

    | E | F | G + | H | I + | J + | K | L | M | N @@ -56,11 +60,192 @@

    Index

    | W
    +

    Symbols

    + + + +
    +

    _

    + -
    -
  • __hash__ (celpy.celtypes.ListType attribute) +
  • __hash__ (celpy.c7nlib.ComparableVersion attribute)
  • __hash__() (celpy.celtypes.BoolType method) @@ -151,6 +348,8 @@

    _

  • (celpy.__init__.Environment method)
  • (celpy.__init__.Runner method) +
  • +
  • (celpy.c7nlib.C7NContext method)
  • (celpy.celparser.CELParseError method)
  • @@ -161,8 +360,6 @@

    _

  • (celpy.celtypes.MapType method)
  • (celpy.celtypes.MessageType method) -
  • -
  • (celpy.celtypes.TypeType method)
  • (celpy.evaluation.Activation method)
  • @@ -173,12 +370,18 @@

    _

  • (celpy.evaluation.CELUnsupportedError method)
  • (celpy.evaluation.Evaluator method) -
  • -
  • (celpy.evaluation.FindIdent method)
  • (celpy.evaluation.NameContainer method) +
  • +
  • (celpy.evaluation.Phase1Transpiler method) +
  • +
  • (celpy.evaluation.Phase2Transpiler method)
  • (celpy.evaluation.Referent method) +
  • +
  • (celpy.evaluation.Transpiler method) +
  • +
  • (celpy.evaluation.TranspilerTree method)
  • __le__() (celpy.celtypes.IntType method) @@ -227,6 +430,8 @@

    _

  • (celpy.celtypes.UintType method)
  • +
    -
  • __repr__() (celpy.celtypes.BoolType method) +
  • __repr__() (celpy.__init__.Environment method)
      +
    • (celpy.__init__.Runner method) +
    • +
    • (celpy.c7nlib.C7NContext method) +
    • +
    • (celpy.celtypes.BoolType method) +
    • (celpy.celtypes.BytesType method)
    • (celpy.celtypes.DoubleType method) @@ -417,31 +632,63 @@

      _

      A

      - +

      B

      + -
      • build_reduce_macro_eval() (celpy.evaluation.Evaluator method)
      • build_ss_macro_eval() (celpy.evaluation.Evaluator method) @@ -463,21 +708,72 @@

        B

        C

        + -
          +
        • + celpy.adapter + +
        • +
        • + celpy.c7nlib + +
        • @@ -505,8 +817,6 @@

          C

        • module
        -
        +
      • ComparableVersion (class in celpy.c7nlib) +
      • compile() (celpy.__init__.Environment method)
      • CompiledRunner (class in celpy.__init__) @@ -536,42 +850,86 @@

        C

      • conditionalor() (celpy.celparser.DumpAST method)
      • +
      • contains() (celpy.c7nlib.IPv4Network method) + +
      • +
      • credentials() (in module celpy.c7nlib) +
      • D

        + -
        • do_set() (celpy.__main__.CEL_REPL method)
        • do_show() (celpy.__main__.CEL_REPL method)
        • dot_ident() (celpy.celparser.DumpAST method) + +
        • dot_ident_arg() (celpy.celparser.DumpAST method) + +
        • DoubleType (class in celpy.celtypes)
        • DumpAST (class in celpy.celparser) @@ -584,8 +942,21 @@

          D

          E

          • Evaluator (class in celpy.evaluation)
          • +
          • + expr + +
          • expr() (celpy.celparser.DumpAST method)
          • exprlist() (celpy.celparser.DumpAST method)
          • extended_name_path (celpy.evaluation.NameContainer attribute) @@ -624,23 +1012,74 @@

            E

            F

            +
            + +

            J

            + + + +
            + +

            K

            + + +
            @@ -729,13 +1268,19 @@

            L

            - +
          • logical_and() (in module celpy.celtypes) @@ -766,50 +1313,82 @@

            L

            M

            + - - +
          • multiplication() (celpy.celparser.DumpAST method)
          • multiplication_div() (celpy.celparser.DumpAST method) @@ -860,6 +1451,8 @@

            N

          • nested_activation() (celpy.evaluation.Activation method)
          • new_activation() (celpy.__init__.Runner method) +
          • +
          • normalize() (in module celpy.c7nlib)
          • NullType (class in celpy.celtypes)
          • @@ -880,23 +1473,64 @@

            P

          • PackageType (class in celpy.celtypes)
          • paren_expr() (celpy.celparser.DumpAST method) + +
          • parent_iter() (celpy.evaluation.NameContainer method)
          • parse() (celpy.celparser.CELParser method)
          • -
            @@ -909,6 +1543,8 @@

            R

          • relation_eq() (celpy.celparser.DumpAST method) @@ -917,19 +1553,25 @@

            R

          • relation_gt() (celpy.celparser.DumpAST method)
          • -
            • relation_le() (celpy.celparser.DumpAST method)
            • relation_lt() (celpy.celparser.DumpAST method)
            • relation_ne() (celpy.celparser.DumpAST method) +
            • +
            • resolve_function() (celpy.evaluation.Activation method)
            • resolve_name() (celpy.evaluation.NameContainer method)
            • resolve_variable() (celpy.evaluation.Activation method) +
            • +
            • resource_schedule() (in module celpy.c7nlib) +
            • +
            • result() (in module celpy.evaluation)
            • Runner (class in celpy.__init__)
            • @@ -940,14 +1582,37 @@

              S

              @@ -955,18 +1620,34 @@

              S

              T

              @@ -998,6 +1681,8 @@

              U

            • unary_neg() (celpy.celparser.DumpAST method)
            • unary_not() (celpy.celparser.DumpAST method) +
            • +
            • unique_size() (in module celpy.c7nlib)
            • @@ -1006,18 +1691,30 @@

              V

              W

              +
              • with_traceback() (celpy.evaluation.CELEvalError method)
              • @@ -1051,14 +1748,15 @@

                CEL in Python

                Navigation

                -

                Contents:

                +

                Documentation Content:

                diff --git a/docs/build/html/index.html b/docs/build/html/index.html index b243b79..bb3edbb 100644 --- a/docs/build/html/index.html +++ b/docs/build/html/index.html @@ -46,44 +46,52 @@

                Pure Python Google Common Expression Language (CEL)

                This implementation has minimal dependencies, runs quickly, and can be embedded into Python-based applications. Specifically, one intent is to be part of Cloud Custodian (C7N) as part of the security policy filter.

                -

                Interested in the API? There are three interesting topics:

                -
                -

                The integration into another application isn’t a trivial import.

                +
                +

                Integration Overview

                +

                Interested in the API for using this package? There are three key topics:

                + +

                The integration into another application is often a bit more than an import. +This is because it involves combining CEL into another DSL.

                +

                The current implementation includes Cloud Custodian (C7N) integration.

                +

                Indices and tables

                  @@ -137,14 +157,15 @@

                  CEL in Python

                  Navigation

                  -

                  Contents:

                  +

                  Documentation Content:

                  diff --git a/docs/build/html/installation.html b/docs/build/html/installation.html index de97ed5..9bde617 100644 --- a/docs/build/html/installation.html +++ b/docs/build/html/installation.html @@ -84,13 +84,13 @@

                  CEL in Python

                  Navigation

                  -

                  Contents:

                  +

                  Documentation Content:

                  diff --git a/docs/build/html/integration.html b/docs/build/html/integration.html index dffb9cb..cbe3e65 100644 --- a/docs/build/html/integration.html +++ b/docs/build/html/integration.html @@ -14,7 +14,7 @@ - + @@ -35,24 +35,20 @@

                  Application Integration

                  -

                  We’ll look at the essential base case for integration: -evaluate a function given some variable bindings.

                  -

                  Then we’ll look at providing custom function bindings to extend -the environment.

                  -

                  We’ll also look at additional examples from the Go implementation. -This will lead us to providing custom type providers -and custom type adapters.

                  -

                  There are a few exception and error-handling cases that are helpful -for writing CEL that tolerates certain kinds of data problems.

                  -

                  Finally, we’ll look at how CEL can be integrated into Cloud Custodian.

                  -
                  -

                  The Essentials

                  -

                  Here are two examples of variable bindings

                  -
                  -

                  README

                  -

                  Here’s the example taken from the README.

                  -

                  The baseline implementation works like this:

                  -
                  >>> import celpy
                  +

                  We’ll look at integration of CEL into another application from four perspectives:

                  +
                    +
                  1. We’ll look at the essential base case for integration into another application. +This will use an Activation to provide values for variables.

                  2. +
                  3. A more sophisticated integration involves extending the environment with custom functions. +This can provide a well-defined interface between CEL expressions and application functionality.

                  4. +
                  5. Some additional examples from the Go implementation show how extend the environment using new types.

                  6. +
                  7. There are a few exception and error-handling cases that are part of working with Python types.

                  8. +
                  9. Finally, we’ll look at how CEL can be integrated into Cloud Custodian (C7N).

                  10. +
                  +
                  +

                  Integration Essentials

                  +

                  Here’s an example of variable bindings taken from a README example:

                  + -

                  The celpy.Environment can include type adapters and type providers. It’s not clear -how these should be implemented in Python or if they’re even necessary.

                  -

                  The compile step creates a syntax tree, which is used to create a final program to evaluate. -Currently, there’s a two-step process because we might want to optimize or transform the AST prior -to evaluation.

                  -

                  The activation provides specific variable values used to evaluate the program.

                  -

                  To an extent, the Python classes are loosely based on the object model in https://github.com/google/cel-go. +

                  The cel_source is an expression to be evaluated. +This references variables with names like account, and transaction.

                  +

                  All CEL evaluation uses an celpy.Environment object. +The celpy.Environment is used to provide type annotations for variables. +It can provide a few other properties, including an overall package name, sometimes needed when working with protobuf types.

                  +

                  The Environment.compile() method creates a abstract syntax tree from the CEL source. +This will be used to create a final program to evaluate. +This method can raise the CELSyntaxError exception.

                  +

                  The Environment.program() method creates a runner out of an abstract syntax tree.

                  +

                  Compiling and building a program is a two-step process to permit optimization or some other transformation the AST prior to evaluation. +The Lark parser (https://lark-parser.readthedocs.io/en/latest/classes.html) is used, and transformers are a first-class feature of this parser.

                  +

                  The context mapping defines variables and provides their values. +This is used to evaluate the resulting program object. +The program will produce a value defined in the celpy.celtypes module. +In this example, it’s a celpy.celtypes.BoolType value.

                  +

                  The CEL types are all specializations of the obvious Python base types. +To an extent, these Python classes are partially based on the object model in https://github.com/google/cel-go. We don’t need all the Go formalisms, however, and rely on Pythonic variants.

                  -
                  -
                  -

                  Simple example using builtin operators

                  +
                  +

                  Simple example using builtin types

                  Here’s an example taken from -https://github.com/google/cel-go/blob/master/examples/README.md

                  -

                  Evaluate expression "Hello world! I'm " + name + "." with CEL passed as -the name variable.

                  +https://github.com/google/cel-go/blob/master/examples/README.md. +This will evaluate the CEL expression "Hello world! I'm " + name + "." with "CEL" passed as the name variable.

                  +

                  This is the original Go code:

                  import (
                       "github.com/google/cel-go/cel"
                       "github.com/google/cel-go/checker/decls"
                  @@ -108,8 +113,8 @@ 

                  Simple example using builtin operators// Output:Hello world! I'm CEL.

                  -

                  Here’s the Python version:

                  -
                  >>> import celpy
                  +

                  Here’s the Python version, following a similar outline:

                  +
                  -

                  There’s a big open concern here: there’s no formal type adapter implementation. -Nothing converts from the input value in the activation to the proper underlying -type. This relies on Python’s built-in type conversions.

                  +

                  The steps include:

                  +
                    +
                  1. Create a celpy.Environment with annotations for any variables. +These kinds of type definitions are atypical for Python, but are part of the definition of the CEL language.

                  2. +
                  3. Use celpy.Environment.compile() to create an AST.

                  4. +
                  5. Use celpy.Environment.program() to build a celpy.Runner object that will do the final evaluation. This includes the environment and the AST.

                  6. +
                  7. Use celpy.Runner.evaluate() to evaluate the program with specific values for the defined variables.

                  8. +
                  +

                  In the Go world, there’s a formal type adapter to convert input values to the objects used by CEL. +For numerous types, a default adapter handles this.

                  +

                  In Python, on the other hand, we define the type conversions as features of the Python versions of the CEL types. +This approach fits better with native Python programming.

                  Function Bindings

                  -

                  Here are two more examples of binding, taken from -https://github.com/google/cel-go/blob/master/examples/README.md

                  -

                  Note the complication here comes from the way the Go implementation resolves overloaded functions. -Each CEL overload of a function is described by a ("name", [args], result) structure. +

                  There are two function binding examples in +https://github.com/google/cel-go/blob/master/examples/README.md.

                  +

                  There is a complication here that based on the way the Go resolves overloaded functions. +In Go, each overload of a function is described by a ("name", [args], result) data structure. +The key of ("name", [args], result) maps to a specific arg_name_arg() or name_arg() overloaded implementation for specific argument types. This allows for multiple type-specific overload versions of a generic function.

                  -

                  The key of ("name", [args], result) maps to a specific arg_name_arg() or name_arg() -overloaded implementation for specific argument types.

                  -

                  For example, ("greet", [StringType, StringType], StringType) maps to string_greet_string().

                  -

                  This is emphatically not how Python generally works. A more Pythonic approach is to provide -a single, generic, function which examines the arguments and decides what to do. Python doesn’t -generally do overloaded name resolution.

                  -

                  There are two choices:

                  -
                    -
                  1. Build a mapping from ("name", [args], result) to a specific overloaded implementation. -This pulls argument and result type coercion outside the Python function. -It matches the Go implementation, but can be confusing for Python implementers. -This requires exposing a great deal of machinery already available in a Python function -definition.

                  2. -
                  3. Ignore the complex type exposture techniques that Go requiees and dispatch to a Python function. -The Python function will sort out type variants and handle argument value coercion on its own. -This simplifies implementation down to name resolution. -Indeed, the type mapping rules can introspect Python’s type annotations on the function -definition.

                  4. -
                  -

                  We follow the 2nd alternative. The Python function binding relies – exclusively – on introspection -of the function provided.

                  -
                  -

                  Custom function on string type

                  -

                  Evaluate expression i.greet(you) with:

                  +

                  For example, a ("greet", [StringType, StringType], StringType) structure is expected to map to a function string_greet_string() that has the expected signature.

                  +

                  This is emphatically not how Python generally works. +We follow a more Pythonic approach is to provide a single, generic, function which examines the arguments and decides what to do. +Outside type-checking, Python doesn’t depend on overloaded name resolution.

                  +

                  This means a Python function must then sort out type variants and handle argument value coercion on its own. +For most cases, the match/case statement is helpful for this. +The functools.singledispatch() decorator can also be helpful for this.

                  +

                  The two examples have slightly different approaches to the CEL expression. +These are important in Go, but less important in Python.

                  +
                  +

                  Custom function in Go

                  +

                  We want to evaluate the CEL expression i.greet(you) with:

                  i       -> CEL
                   you     -> world
                   greet   -> "Hello %s! Nice to meet you, I'm %s."
                   
                  -

                  First we need to declare two string variables and greet function. -NewInstanceOverload must be used if we want to declare function which will -operate on a type. First element of slice passed as argTypes into -NewInstanceOverload is declaration of instance type. Next elements are -parameters of function.

                  +

                  The idea here is the new greet() behaves like a method of a String. +The actual implementation, however, is not a method; it’s a function of two arguments.

                  +

                  First we need to declare two string variables and a greet() function. +In Go, a NewInstanceOverload must be used to provide annotations for variables and the function. +Here’s the Go implementation:

                  decls.NewVar("i", decls.String),
                   decls.NewVar("you", decls.String),
                   decls.NewFunction("greet",
                  @@ -182,9 +185,12 @@ 

                  Custom function on string type... // Create env and compile

                  -

                  Let’s implement greet function and pass it to program. We will be using -Binary, because greet function uses 2 parameters (1st instance, 2nd -function parameter).

                  +

                  We’ve omitted the Go details of creating an environment and compiling the CEL expression. +These aren’t different from the previous examples.

                  +

                  Separately, a greetFunc() function must be defined. +In Go, this function is then bound to the "string_greet_string" overload, +ready for evaluation. +Here’s the Go implementation:

                  -

                  Here’s the Python version:

                  -
                  >>> import celpy
                  +

                  What’s essential is defining some type information, then defining variables and functions that fit those types.

                  +

                  The Python version has the same outline:

                  +
                    +
                  1. An celpy.Environment with type annotations for the two variables and the function.

                  2. +
                  3. Compile the source.

                  4. +
                  +

                  3. Define the greet() function. While the CEL syntax of i.greet(you) looks like a method +of the i variable’s class, the function is simply has two positional parameters.

                  +
                    +
                  1. Provide function implementation when creating the final celpy.Runner instance.

                  2. +
                  3. Evaluate the program with specific values for the two variables.

                  4. +
                  +
                  +

                  The key concept here is to distinguish between three distinct attributes:

                  +
                    +
                  1. Type annotations associated with variables or functions.

                  2. +
                  3. The function implementations used to build the celpy.Runner. +The method-like syntax of i.greet(you) is evaluated as greet(i, you).

                  4. +
                  5. The variable values, which provide a context in which the runner evaluates the CEL expression.

                  6. +
                  +

                  This reflects the idea that one CEL expression may be used to process data over and over again.

                  Define custom global function

                  -

                  Evaluate expression shake_hands(i,you) with:

                  -
                  i           -> CEL
                  -you         -> world
                  -shake_hands -> "%s and %s are shaking hands."
                  -
                  -
                  -

                  In order to declare global function we need to use NewOverload:

                  -
                  decls.NewVar("i", decls.String),
                  -decls.NewVar("you", decls.String),
                  -decls.NewFunction("shake_hands",
                  -    decls.NewOverload("shake_hands_string_string",
                  -        []*exprpb.Type{decls.String, decls.String},
                  -        decls.String))
                  -... // Create env and compile.
                  -
                  -shakeFunc := &functions.Overload{
                  -    Operator: "shake_hands_string_string",
                  -    Binary: func(lhs ref.Val, rhs ref.Val) ref.Val {
                  -        return types.String(
                  -            fmt.Sprintf("%s and %s are shaking hands.\n", lhs, rhs))
                  -        }}
                  -prg, err := env.Program(c, cel.Functions(shakeFunc))
                  -
                  -out, _, err := prg.Eval(map[string]interface{}{
                  -    "i": "CEL",
                  -    "you": "world",
                  -})
                  -fmt.Println(out)
                  -// Output:CEL and world are shaking hands.
                  -
                  -
                  +

                  In Go, this is a small, but important different.ce +We want to evaluate the expression shake_hands(i,you). +This uses a global function syntax instead of method syntax.

                  +

                  While Go has slight differences in how the function is defined, in Python, there is no change.

                  Here’s the Python version:

                  -
                  -
                  -

                  Examples from Go implementation

                  -

                  See https://github.com/google/cel-go/blob/master/README.md

                  +
                  +

                  More Examples from Go implementation

                  +

                  See https://github.com/google/cel-go/blob/master/README.md for five more examples.

                  // Check whether a resource name starts with a group name.
                   resource.name.startsWith("/groups/" + auth.claims.group)
                   
                  @@ -305,7 +304,8 @@ 

                  Examples from Go implementationhas(message.field)

                  -

                  Following one of the more complete examples through the README

                  +

                  Here’s the first example, resource.name.startsWith("/groups/" + auth.claims.group). +The Go code is as follows:

                  -

                  This has the following Python implementation:

                  -

                  Exceptions and Errors

                  -

                  Exceptions raised in Python world will (eventually) crash the CEL evluation. -This gives the author of an extension function the complete traceback to help -fix the Python code. +

                  Exceptions raised in Python world will (eventually) crash the CEL evaluation. +This gives the author of an extension function the complete traceback to help fix the Python code. No masking or rewriting of Python exceptions ever occurs in extension functions.

                  -

                  A special celpy.EvalError exception can be used in an extension function -to permit CEL’s short-circuit logic processing to silence this exception. See the -https://github.com/google/cel-go/blob/master/README.md#partial-state for more examples -of how the short-circuit (partial state) operations work.

                  -

                  An extension function must return a celpy.EvalError object -to allow processing to continue in spite of an uncomputable value.

                  -
                  from celpy import *
                  +

                  A special celpy.CELEvalError exception can be used in an extension function to permit CEL’s short-circuit logic processing to check and ignore an exception. +See the https://github.com/google/cel-go/blob/master/README.md#partial-state for more examples of how the short-circuit (partial state) operations work.

                  +

                  An extension function can return a celpy.CELEvalError object instead of raising it. +This can allow processing to continue in spite of an uncomputable value.

                  +
                  from celpy import *
                   def my_extension(a: Value) -> Value:
                       try:
                           return celtypes.UintType(64 // a)
                       except DivideByZeroError as ex:
                  -        return EvalError(f"my_extnsion({a}) error")
                  -
                  -
                  -

                  The returned exception object allows short-circuit processing. For example,

                  -
                  false && my_extension(0)
                  -
                  -
                  -

                  This evaluates to false. If computed, any celpy.EvalError object will be silently ignored.

                  -

                  On the other hand,

                  -
                  true && my_extension(0)
                  +        return CELEvalError(f"my_extension({a}) error")
                   
                  -

                  This will result in a visible celpy.EvalError result from the extension function. +

                  The returned exception object allows short-circuit processing. +For example, the CEL expression false && my_extension(0) evaluates to false. +If computed, any celpy.CELEvalError objects will be silently ignored because the short-circuit result is known from the presence of a false value.

                  +

                  On the other hand, the CEL expression true && my_extension(0) results in the celpy.CELEvalError result from the extension function. This will eventually be raised as an exception, so the framework using celpy can track this run-time error.

                  -
                  -

                  Cloud Custodian

                  -

                  Custodian Filters can be evaluated by CEL.

                  +
                  +

                  Cloud Custodian (C7N) Integration

                  +

                  Custodian Filters can be evaluated by CEL. +The idea is to extend the YAML-based DSL for policy documents to introduce easier-to-read expressions.

                  As noted in https://github.com/cloud-custodian/cloud-custodian/issues/5759, a filter might look like the following:

                  filters:
                  @@ -402,13 +396,11 @@ 

                  Cloud Custodian-  and: and -  or: sub-documents with a CEL expression.

                  -

                  C7N processes resources by gathering resources, creating an instance of a subclass of the Filter -class, and evaluating an expression like take_action = list(filter(filter_instance, resource_list)).

                  -

                  The C7N filter expression in a given policy document is componsed of one or more atomic filter clauses, -combined by and, or, and not operators. +

                  C7N processioning works by gathering resources, creating an instance of a subclass of the Filter class, and evaluating an expression like take_action = list(filter(filter_instance, resource_list)).

                  +

                  The C7N filter expression in a given policy document is composed of one or more atomic filter clauses, combined by and, or, and not operators. The filter as a whole is handled by the __call__() methods of subclasses of the BooleanGroupFilter class.

                  Central to making this work is making the CEL expression into a function that can be applied to the resource object. -It appears that all CEL operations will need to have a number of values in their activations:

                  +All CEL versions of a filter will need to have a the following two values in their activations:

                  resource:

                  A celtypes.MapType document with the resource details.

                  @@ -417,12 +409,11 @@

                  Cloud Custodian

                  A celtypes.TimestampType object with the current time.

                  -

                  Additional “global” objects may also be helpful.

                  Baseline C7N Example

                  -

                  The essence of the integration is to provide a resource to a function and receive a boolean result.

                  +

                  The essence of the integration is to provide a resource description to a function defined as a CEL expression, and receive a boolean result.

                  Here’s a base example:

                  -
                  >>> import celpy
                  +
                  >>> import celpy
                   >>> env = celpy.Environment()
                   >>> CEL = """
                   ... resource.creationTimestamp < timestamp("2018-08-03T16:00:00-07:00") &&
                  @@ -450,6 +441,8 @@ 

                  Baseline C7N ExampleBoolType(True)

                  +

                  In this case, the context contained only one variable, resource. +It didn’t require a definition of now.

                  Bulk Filter Example

                  @@ -458,8 +451,9 @@

                  Bulk Filter Examplemap(action, list(filter(cel_program, resources)))

                  -

                  An action is applied to those resources that pass some filter test. The filter looks for items not compliant -with policies.

                  +

                  An action is applied to those resources that pass some filter test. +Often, the action disables a resource to prevent data compromise. +The filter looks for items not compliant with policies so they can be deleted or disabled.

                  The cel_program in the above example is an executable CEL program wrapped into a C7N Filter subclass.

                  +

                  For each resource, the tag_policy_filter object applied an internal self.prgm to the resource. +The internal self.prgm was built from the policy expression, stated in CEL.

                  C7N Filter and Resource Types

                  -

                  There are several parts to handling the various kinds of C7N filters in use.

                  +

                  The celpy.c7nlib module provides filter subclasses that include CEL processing. +There are two kinds of C7N filters in use.

                    -
                  1. The c7n.filters package defines about 23 generic filter classes, all of which need to -provide the resource object in the activation, and possibly provide a library of generic -CEL functions used for evaluation. -The general cases are of this is handled by the resource definition classes creating values in a JSON document. +

                  2. The c7n.filters package defines about 23 generic filter classes. +These apply to a resource object. +Additionally, there’s a library of generic functions used for evaluation. +Generally, the resource definition classes create values in a JSON document. These values reflect the state of the resource and any closely-related resources.

                  3. The c7n.resources package defines a number of additional resource-specific filters. -All of these, similarly, need to provide CEL values as part of the resource object. -These classes can also provide additional resource-specific CEL functions used for evaluation.

                  4. +These classes can also provide additional resource-specific processing.

                  -

                  The atomic filter clauses have two general forms:

                  +

                  The atomic filter clauses within a policy document have two general forms:

                  Navigation

                  -

                  Contents:

                  +

                  Documentation Content:

                  diff --git a/docs/build/html/search.html b/docs/build/html/search.html index a97df35..29f21dd 100644 --- a/docs/build/html/search.html +++ b/docs/build/html/search.html @@ -81,14 +81,15 @@

                  CEL in Python

                  Navigation

                  -

                  Contents:

                  +

                  Documentation Content:

                  diff --git a/docs/build/html/searchindex.js b/docs/build/html/searchindex.js index d20657c..5e35916 100644 --- a/docs/build/html/searchindex.js +++ b/docs/build/html/searchindex.js @@ -1 +1 @@ -Search.setIndex({"alltitles":{"Another Example":[[0,"another-example"]],"Application Integration":[[6,null]],"Arguments, Types, and Namespaces":[[0,"arguments-types-and-namespaces"]],"Baseline C7N Example":[[6,"baseline-c7n-example"]],"Bulk Filter Example":[[6,"bulk-filter-example"]],"C7N Cache":[[1,"c7n-cache"]],"C7N Context Object":[[1,"c7n-context-object"]],"C7N Filter and Resource Types":[[6,"c7n-filter-and-resource-types"]],"C7N Functions Required":[[1,null]],"CEL Types":[[7,"cel-types"]],"CEL-Py API":[[0,null]],"CELFilter Design":[[1,"celfilter-design"]],"CLI Use of CEL-Python":[[2,null]],"Cloud Custodian":[[6,"cloud-custodian"]],"Common C7N Constructs":[[1,"common-c7n-constructs"]],"Common/Boolean Filters":[[1,"common-boolean-filters"]],"Common/Non-Bool Filters":[[1,"common-non-bool-filters"]],"Configuration":[[3,null]],"Contents":[[1,"contents"]],"Contents:":[[4,null]],"Custom function on string type":[[6,"custom-function-on-string-type"]],"Data Structures":[[7,null]],"Define custom global function":[[6,"define-custom-global-function"]],"Design Principles":[[1,"design-principles"]],"Example":[[0,"example"]],"Examples from Go implementation":[[6,"examples-from-go-implementation"]],"Exceptions and Errors":[[6,"exceptions-and-errors"]],"Function Bindings":[[6,"function-bindings"]],"Indices and tables":[[4,"indices-and-tables"]],"Installation":[[5,null]],"Numeric Details":[[0,"numeric-details"]],"Pure Python Google Common Expression Language (CEL)":[[4,null]],"README":[[6,"readme"]],"Run-Time":[[7,"run-time"]],"SYNOPSIS":[[0,"synopsis"]],"Simple example using builtin operators":[[6,"simple-example-using-builtin-operators"]],"Singleton/Boolean Filters":[[1,"singleton-boolean-filters"]],"Singleton/Non-Bool Filters":[[1,"singleton-non-bool-filters"]],"Summary":[[1,"summary"]],"The Essentials":[[6,"the-essentials"]],"Timzone Details":[[0,"timzone-details"]],"Todo":[[0,"id1"],[0,"id2"],[0,"id3"],[0,"id4"],[0,"id5"],[0,"id6"],[0,"id7"],[0,"id8"],[0,"id9"],[0,"id10"],[0,"id11"],[0,"id12"],[0,"id13"],[6,"id1"]],"Type Adapter":[[0,"type-adapter"]],"Type Provider":[[0,"type-provider"]],"Types":[[0,"types"]],"__main__":[[0,"module-celpy.__main__"]],"access-key":[[1,"access-key"]],"age":[[1,"age"]],"bucket-encryption (no examples)":[[1,"bucket-encryption-no-examples"]],"bucket-notification (no examples)":[[1,"bucket-notification-no-examples"]],"capacity-delta":[[1,"capacity-delta"]],"celtypes":[[0,"module-celpy.celtypes"]],"check-cloudtrail":[[1,"check-cloudtrail"]],"check-config":[[1,"check-config"]],"config-compliance (no examples)":[[1,"config-compliance-no-examples"]],"credential":[[1,"credential"]],"cross-account":[[1,"cross-account"]],"data-events (no examples)":[[1,"data-events-no-examples"]],"db-parameter (no examples)":[[1,"db-parameter-no-examples"]],"default-vpc (no examples)":[[1,"default-vpc-no-examples"]],"dhcp-options (no examples)":[[1,"dhcp-options-no-examples"]],"diff (no examples)":[[1,"diff-no-examples"]],"ebs":[[1,"ebs"]],"egress":[[1,"egress"]],"ephemeral (no examples)":[[1,"ephemeral-no-examples"]],"evaluation":[[0,"module-celpy.evaluation"]],"event":[[1,"event"]],"event-source (no examples)":[[1,"event-source-no-examples"]],"fault-tolerant (no examples)":[[1,"fault-tolerant-no-examples"]],"finding (no examples)":[[1,"finding-no-examples"]],"flow-logs":[[1,"flow-logs"]],"global-grants":[[1,"global-grants"]],"grant-count":[[1,"grant-count"]],"group (no examples)":[[1,"group-no-examples"]],"has-allow-all (no examples)":[[1,"has-allow-all-no-examples"]],"has-inline-policy (no examples)":[[1,"has-inline-policy-no-examples"]],"has-specific-managed-policy (no examples)":[[1,"has-specific-managed-policy-no-examples"]],"has-statement":[[1,"has-statement"]],"has-users (no examples)":[[1,"has-users-no-examples"]],"has-virtual-mfa (no examples)":[[1,"has-virtual-mfa-no-examples"]],"health-event":[[1,"health-event"]],"healthcheck-protocol-mismatch (no examples)":[[1,"healthcheck-protocol-mismatch-no-examples"]],"iam-summary (no examples)":[[1,"iam-summary-no-examples"]],"image":[[1,"image"]],"image-age":[[1,"image-age"]],"ingress":[[1,"ingress"]],"instance (no examples)":[[1,"instance-no-examples"]],"instance-age":[[1,"instance-age"]],"instance-attribute (no examples)":[[1,"instance-attribute-no-examples"]],"instance-uptime":[[1,"instance-uptime"]],"invalid":[[1,"invalid"]],"inventory (no examples)":[[1,"inventory-no-examples"]],"is-log-target":[[1,"is-log-target"]],"is-logging":[[1,"is-logging"]],"is-not-logging":[[1,"is-not-logging"]],"is-shadow (no examples)":[[1,"is-shadow-no-examples"]],"is-ssl (no examples)":[[1,"is-ssl-no-examples"]],"key-rotation-status":[[1,"key-rotation-status"]],"kms-alias":[[1,"kms-alias"]],"kms-key":[[1,"kms-key"]],"last-write":[[1,"last-write"]],"latest":[[1,"latest"]],"launch-config":[[1,"launch-config"]],"lifecycle-rule (no examples)":[[1,"lifecycle-rule-no-examples"]],"listener":[[1,"listener"]],"locked (no examples)":[[1,"locked-no-examples"]],"marked-for-op":[[1,"marked-for-op"]],"metrics":[[1,"metrics"]],"mfa-device":[[1,"mfa-device"]],"mismatch-s3-origin":[[1,"mismatch-s3-origin"]],"missing":[[1,"missing"]],"missing-policy-statement":[[1,"missing-policy-statement"]],"missing-route (no examples)":[[1,"missing-route-no-examples"]],"modifyable (no examples)":[[1,"modifyable-no-examples"]],"network-location":[[1,"network-location"]],"no-encryption-statement (no examples)":[[1,"no-encryption-statement-no-examples"]],"no-specific-managed-policy (no examples)":[[1,"no-specific-managed-policy-no-examples"]],"not-encrypted":[[1,"not-encrypted"]],"offhour":[[1,"offhour"]],"onhour":[[1,"onhour"]],"op Implementations":[[1,"op-implementations"]],"param (no examples)":[[1,"param-no-examples"]],"parser":[[0,"module-celpy.celparser"]],"password-policy":[[1,"password-policy"]],"policy":[[1,"policy"]],"progagated-tags (no examples)":[[1,"progagated-tags-no-examples"]],"query-logging-enabled (no examples)":[[1,"query-logging-enabled-no-examples"]],"reserved-concurrency":[[1,"reserved-concurrency"]],"rest-integration (no examples)":[[1,"rest-integration-no-examples"]],"rest-method (no examples)":[[1,"rest-method-no-examples"]],"route (no examples)":[[1,"route-no-examples"]],"s3-cidr (no examples)":[[1,"s3-cidr-no-examples"]],"s3-public-block (no examples)":[[1,"s3-public-block-no-examples"]],"security-group":[[1,"security-group"]],"service-limit":[[1,"service-limit"]],"shield-enabled":[[1,"shield-enabled"]],"shield-metrics (no examples)":[[1,"shield-metrics-no-examples"]],"singleton (no examples)":[[1,"singleton-no-examples"]],"skip-ami-snapshots":[[1,"skip-ami-snapshots"]],"ssl-policy":[[1,"ssl-policy"]],"ssm (no examples)":[[1,"ssm-no-examples"]],"stale (no examples)":[[1,"stale-no-examples"]],"state-age":[[1,"state-age"]],"status (no examples)":[[1,"status-no-examples"]],"subnet":[[1,"subnet"]],"tag-count":[[1,"tag-count"]],"target-group (no examples)":[[1,"target-group-no-examples"]],"task-definition (no examples)":[[1,"task-definition-no-examples"]],"termination-protected (no examples)":[[1,"termination-protected-no-examples"]],"unused":[[1,"unused"]],"upgrade-available (no examples)":[[1,"upgrade-available-no-examples"]],"used":[[1,"used"]],"user-data (no examples)":[[1,"user-data-no-examples"]],"valid":[[1,"valid"]],"value":[[1,"value"]],"value_from External Data":[[1,"value-from-external-data"]],"value_type Conversions":[[1,"value-type-conversions"]],"vpc":[[1,"vpc"]],"vpc-attributes (no examples)":[[1,"vpc-attributes-no-examples"]],"vpc-id":[[1,"vpc-id"]],"waf-enabled":[[1,"waf-enabled"]],"xray-encrypt-key (no examples)":[[1,"xray-encrypt-key-no-examples"]]},"docnames":["api","c7n_functions","cli","configuration","index","installation","integration","structure"],"envversion":{"sphinx":65,"sphinx.domains.c":3,"sphinx.domains.changeset":1,"sphinx.domains.citation":1,"sphinx.domains.cpp":9,"sphinx.domains.index":1,"sphinx.domains.javascript":3,"sphinx.domains.math":2,"sphinx.domains.python":4,"sphinx.domains.rst":2,"sphinx.domains.std":2,"sphinx.ext.todo":2,"sphinx.ext.viewcode":1},"filenames":["api.rst","c7n_functions.rst","cli.rst","configuration.rst","index.rst","installation.rst","integration.rst","structure.rst"],"indexentries":{"__abstractmethods__ (celpy.evaluation.evaluator attribute)":[[0,"celpy.evaluation.Evaluator.__abstractmethods__",false]],"__add__() (celpy.celtypes.durationtype method)":[[0,"celpy.celtypes.DurationType.__add__",false]],"__add__() (celpy.celtypes.inttype method)":[[0,"celpy.celtypes.IntType.__add__",false]],"__add__() (celpy.celtypes.timestamptype method)":[[0,"celpy.celtypes.TimestampType.__add__",false]],"__add__() (celpy.celtypes.uinttype method)":[[0,"celpy.celtypes.UintType.__add__",false]],"__add__() (celpy.evaluation.celevalerror method)":[[0,"celpy.evaluation.CELEvalError.__add__",false]],"__call__() (celpy.celtypes.functiontype method)":[[0,"celpy.celtypes.FunctionType.__call__",false]],"__call__() (celpy.evaluation.celevalerror method)":[[0,"celpy.evaluation.CELEvalError.__call__",false]],"__eq__() (celpy.celtypes.doubletype method)":[[0,"celpy.celtypes.DoubleType.__eq__",false]],"__eq__() (celpy.celtypes.inttype method)":[[0,"celpy.celtypes.IntType.__eq__",false]],"__eq__() (celpy.celtypes.listtype method)":[[0,"celpy.celtypes.ListType.__eq__",false]],"__eq__() (celpy.celtypes.maptype method)":[[0,"celpy.celtypes.MapType.__eq__",false]],"__eq__() (celpy.celtypes.nulltype method)":[[0,"celpy.celtypes.NullType.__eq__",false]],"__eq__() (celpy.celtypes.stringtype method)":[[0,"celpy.celtypes.StringType.__eq__",false]],"__eq__() (celpy.celtypes.typetype method)":[[0,"celpy.celtypes.TypeType.__eq__",false]],"__eq__() (celpy.celtypes.uinttype method)":[[0,"celpy.celtypes.UintType.__eq__",false]],"__eq__() (celpy.evaluation.celevalerror method)":[[0,"celpy.evaluation.CELEvalError.__eq__",false]],"__floordiv__() (celpy.celtypes.inttype method)":[[0,"celpy.celtypes.IntType.__floordiv__",false]],"__floordiv__() (celpy.celtypes.uinttype method)":[[0,"celpy.celtypes.UintType.__floordiv__",false]],"__floordiv__() (celpy.evaluation.celevalerror method)":[[0,"celpy.evaluation.CELEvalError.__floordiv__",false]],"__ge__() (celpy.celtypes.inttype method)":[[0,"celpy.celtypes.IntType.__ge__",false]],"__ge__() (celpy.celtypes.listtype method)":[[0,"celpy.celtypes.ListType.__ge__",false]],"__getitem__() (celpy.celtypes.maptype method)":[[0,"celpy.celtypes.MapType.__getitem__",false]],"__gt__() (celpy.celtypes.inttype method)":[[0,"celpy.celtypes.IntType.__gt__",false]],"__gt__() (celpy.celtypes.listtype method)":[[0,"celpy.celtypes.ListType.__gt__",false]],"__hash__ (celpy.celtypes.listtype attribute)":[[0,"celpy.celtypes.ListType.__hash__",false]],"__hash__ (celpy.celtypes.maptype attribute)":[[0,"celpy.celtypes.MapType.__hash__",false]],"__hash__ (celpy.celtypes.nulltype attribute)":[[0,"celpy.celtypes.NullType.__hash__",false]],"__hash__ (celpy.celtypes.typetype attribute)":[[0,"celpy.celtypes.TypeType.__hash__",false]],"__hash__ (celpy.evaluation.celevalerror attribute)":[[0,"celpy.evaluation.CELEvalError.__hash__",false]],"__hash__() (celpy.celtypes.booltype method)":[[0,"celpy.celtypes.BoolType.__hash__",false]],"__hash__() (celpy.celtypes.doubletype method)":[[0,"celpy.celtypes.DoubleType.__hash__",false]],"__hash__() (celpy.celtypes.inttype method)":[[0,"celpy.celtypes.IntType.__hash__",false]],"__hash__() (celpy.celtypes.stringtype method)":[[0,"celpy.celtypes.StringType.__hash__",false]],"__hash__() (celpy.celtypes.uinttype method)":[[0,"celpy.celtypes.UintType.__hash__",false]],"__init__() (celpy.__init__.compiledrunner method)":[[0,"celpy.__init__.CompiledRunner.__init__",false]],"__init__() (celpy.__init__.environment method)":[[0,"celpy.__init__.Environment.__init__",false]],"__init__() (celpy.__init__.runner method)":[[0,"celpy.__init__.Runner.__init__",false]],"__init__() (celpy.celparser.celparseerror method)":[[0,"celpy.celparser.CELParseError.__init__",false]],"__init__() (celpy.celparser.celparser method)":[[0,"celpy.celparser.CELParser.__init__",false]],"__init__() (celpy.celparser.dumpast method)":[[0,"celpy.celparser.DumpAST.__init__",false]],"__init__() (celpy.celtypes.maptype method)":[[0,"celpy.celtypes.MapType.__init__",false]],"__init__() (celpy.celtypes.messagetype method)":[[0,"celpy.celtypes.MessageType.__init__",false]],"__init__() (celpy.celtypes.typetype method)":[[0,"celpy.celtypes.TypeType.__init__",false]],"__init__() (celpy.evaluation.activation method)":[[0,"celpy.evaluation.Activation.__init__",false]],"__init__() (celpy.evaluation.celevalerror method)":[[0,"celpy.evaluation.CELEvalError.__init__",false]],"__init__() (celpy.evaluation.celsyntaxerror method)":[[0,"celpy.evaluation.CELSyntaxError.__init__",false]],"__init__() (celpy.evaluation.celunsupportederror method)":[[0,"celpy.evaluation.CELUnsupportedError.__init__",false]],"__init__() (celpy.evaluation.evaluator method)":[[0,"celpy.evaluation.Evaluator.__init__",false]],"__init__() (celpy.evaluation.findident method)":[[0,"celpy.evaluation.FindIdent.__init__",false]],"__init__() (celpy.evaluation.namecontainer method)":[[0,"celpy.evaluation.NameContainer.__init__",false]],"__init__() (celpy.evaluation.referent method)":[[0,"celpy.evaluation.Referent.__init__",false]],"__le__() (celpy.celtypes.inttype method)":[[0,"celpy.celtypes.IntType.__le__",false]],"__le__() (celpy.celtypes.listtype method)":[[0,"celpy.celtypes.ListType.__le__",false]],"__lt__() (celpy.celtypes.inttype method)":[[0,"celpy.celtypes.IntType.__lt__",false]],"__lt__() (celpy.celtypes.listtype method)":[[0,"celpy.celtypes.ListType.__lt__",false]],"__mod__() (celpy.celtypes.doubletype method)":[[0,"celpy.celtypes.DoubleType.__mod__",false]],"__mod__() (celpy.celtypes.inttype method)":[[0,"celpy.celtypes.IntType.__mod__",false]],"__mod__() (celpy.celtypes.uinttype method)":[[0,"celpy.celtypes.UintType.__mod__",false]],"__mod__() (celpy.evaluation.celevalerror method)":[[0,"celpy.evaluation.CELEvalError.__mod__",false]],"__mul__() (celpy.celtypes.inttype method)":[[0,"celpy.celtypes.IntType.__mul__",false]],"__mul__() (celpy.celtypes.uinttype method)":[[0,"celpy.celtypes.UintType.__mul__",false]],"__mul__() (celpy.evaluation.celevalerror method)":[[0,"celpy.evaluation.CELEvalError.__mul__",false]],"__ne__() (celpy.celtypes.doubletype method)":[[0,"celpy.celtypes.DoubleType.__ne__",false]],"__ne__() (celpy.celtypes.inttype method)":[[0,"celpy.celtypes.IntType.__ne__",false]],"__ne__() (celpy.celtypes.listtype method)":[[0,"celpy.celtypes.ListType.__ne__",false]],"__ne__() (celpy.celtypes.maptype method)":[[0,"celpy.celtypes.MapType.__ne__",false]],"__ne__() (celpy.celtypes.nulltype method)":[[0,"celpy.celtypes.NullType.__ne__",false]],"__ne__() (celpy.celtypes.stringtype method)":[[0,"celpy.celtypes.StringType.__ne__",false]],"__ne__() (celpy.celtypes.uinttype method)":[[0,"celpy.celtypes.UintType.__ne__",false]],"__neg__() (celpy.celtypes.booltype method)":[[0,"celpy.celtypes.BoolType.__neg__",false]],"__neg__() (celpy.celtypes.doubletype method)":[[0,"celpy.celtypes.DoubleType.__neg__",false]],"__neg__() (celpy.celtypes.inttype method)":[[0,"celpy.celtypes.IntType.__neg__",false]],"__neg__() (celpy.celtypes.uinttype method)":[[0,"celpy.celtypes.UintType.__neg__",false]],"__neg__() (celpy.evaluation.celevalerror method)":[[0,"celpy.evaluation.CELEvalError.__neg__",false]],"__new__() (celpy.__init__.int32value static method)":[[0,"celpy.__init__.Int32Value.__new__",false]],"__new__() (celpy.celtypes.booltype static method)":[[0,"celpy.celtypes.BoolType.__new__",false]],"__new__() (celpy.celtypes.bytestype static method)":[[0,"celpy.celtypes.BytesType.__new__",false]],"__new__() (celpy.celtypes.doubletype static method)":[[0,"celpy.celtypes.DoubleType.__new__",false]],"__new__() (celpy.celtypes.durationtype static method)":[[0,"celpy.celtypes.DurationType.__new__",false]],"__new__() (celpy.celtypes.inttype static method)":[[0,"celpy.celtypes.IntType.__new__",false]],"__new__() (celpy.celtypes.stringtype static method)":[[0,"celpy.celtypes.StringType.__new__",false]],"__new__() (celpy.celtypes.timestamptype static method)":[[0,"celpy.celtypes.TimestampType.__new__",false]],"__new__() (celpy.celtypes.uinttype static method)":[[0,"celpy.celtypes.UintType.__new__",false]],"__orig_bases__ (celpy.celtypes.listtype attribute)":[[0,"celpy.celtypes.ListType.__orig_bases__",false]],"__orig_bases__ (celpy.celtypes.maptype attribute)":[[0,"celpy.celtypes.MapType.__orig_bases__",false]],"__orig_bases__ (celpy.evaluation.namecontainer attribute)":[[0,"celpy.evaluation.NameContainer.__orig_bases__",false]],"__parameters__ (celpy.celparser.dumpast attribute)":[[0,"celpy.celparser.DumpAST.__parameters__",false]],"__parameters__ (celpy.celtypes.listtype attribute)":[[0,"celpy.celtypes.ListType.__parameters__",false]],"__parameters__ (celpy.celtypes.maptype attribute)":[[0,"celpy.celtypes.MapType.__parameters__",false]],"__parameters__ (celpy.celtypes.messagetype attribute)":[[0,"celpy.celtypes.MessageType.__parameters__",false]],"__parameters__ (celpy.celtypes.packagetype attribute)":[[0,"celpy.celtypes.PackageType.__parameters__",false]],"__parameters__ (celpy.evaluation.evaluator attribute)":[[0,"celpy.evaluation.Evaluator.__parameters__",false]],"__parameters__ (celpy.evaluation.findident attribute)":[[0,"celpy.evaluation.FindIdent.__parameters__",false]],"__parameters__ (celpy.evaluation.namecontainer attribute)":[[0,"celpy.evaluation.NameContainer.__parameters__",false]],"__pow__() (celpy.evaluation.celevalerror method)":[[0,"celpy.evaluation.CELEvalError.__pow__",false]],"__radd__() (celpy.celtypes.durationtype method)":[[0,"celpy.celtypes.DurationType.__radd__",false]],"__radd__() (celpy.celtypes.inttype method)":[[0,"celpy.celtypes.IntType.__radd__",false]],"__radd__() (celpy.celtypes.timestamptype method)":[[0,"celpy.celtypes.TimestampType.__radd__",false]],"__radd__() (celpy.celtypes.uinttype method)":[[0,"celpy.celtypes.UintType.__radd__",false]],"__radd__() (celpy.evaluation.celevalerror method)":[[0,"celpy.evaluation.CELEvalError.__radd__",false]],"__repr__() (celpy.celtypes.booltype method)":[[0,"celpy.celtypes.BoolType.__repr__",false]],"__repr__() (celpy.celtypes.bytestype method)":[[0,"celpy.celtypes.BytesType.__repr__",false]],"__repr__() (celpy.celtypes.doubletype method)":[[0,"celpy.celtypes.DoubleType.__repr__",false]],"__repr__() (celpy.celtypes.durationtype method)":[[0,"celpy.celtypes.DurationType.__repr__",false]],"__repr__() (celpy.celtypes.inttype method)":[[0,"celpy.celtypes.IntType.__repr__",false]],"__repr__() (celpy.celtypes.listtype method)":[[0,"celpy.celtypes.ListType.__repr__",false]],"__repr__() (celpy.celtypes.maptype method)":[[0,"celpy.celtypes.MapType.__repr__",false]],"__repr__() (celpy.celtypes.stringtype method)":[[0,"celpy.celtypes.StringType.__repr__",false]],"__repr__() (celpy.celtypes.timestamptype method)":[[0,"celpy.celtypes.TimestampType.__repr__",false]],"__repr__() (celpy.celtypes.uinttype method)":[[0,"celpy.celtypes.UintType.__repr__",false]],"__repr__() (celpy.evaluation.activation method)":[[0,"celpy.evaluation.Activation.__repr__",false]],"__repr__() (celpy.evaluation.celevalerror method)":[[0,"celpy.evaluation.CELEvalError.__repr__",false]],"__repr__() (celpy.evaluation.namecontainer method)":[[0,"celpy.evaluation.NameContainer.__repr__",false]],"__repr__() (celpy.evaluation.referent method)":[[0,"celpy.evaluation.Referent.__repr__",false]],"__rfloordiv__() (celpy.celtypes.inttype method)":[[0,"celpy.celtypes.IntType.__rfloordiv__",false]],"__rfloordiv__() (celpy.celtypes.uinttype method)":[[0,"celpy.celtypes.UintType.__rfloordiv__",false]],"__rfloordiv__() (celpy.evaluation.celevalerror method)":[[0,"celpy.evaluation.CELEvalError.__rfloordiv__",false]],"__rmod__() (celpy.celtypes.doubletype method)":[[0,"celpy.celtypes.DoubleType.__rmod__",false]],"__rmod__() (celpy.celtypes.inttype method)":[[0,"celpy.celtypes.IntType.__rmod__",false]],"__rmod__() (celpy.celtypes.uinttype method)":[[0,"celpy.celtypes.UintType.__rmod__",false]],"__rmod__() (celpy.evaluation.celevalerror method)":[[0,"celpy.evaluation.CELEvalError.__rmod__",false]],"__rmul__() (celpy.celtypes.inttype method)":[[0,"celpy.celtypes.IntType.__rmul__",false]],"__rmul__() (celpy.celtypes.uinttype method)":[[0,"celpy.celtypes.UintType.__rmul__",false]],"__rmul__() (celpy.evaluation.celevalerror method)":[[0,"celpy.evaluation.CELEvalError.__rmul__",false]],"__rpow__() (celpy.evaluation.celevalerror method)":[[0,"celpy.evaluation.CELEvalError.__rpow__",false]],"__rsub__() (celpy.celtypes.inttype method)":[[0,"celpy.celtypes.IntType.__rsub__",false]],"__rsub__() (celpy.celtypes.uinttype method)":[[0,"celpy.celtypes.UintType.__rsub__",false]],"__rsub__() (celpy.evaluation.celevalerror method)":[[0,"celpy.evaluation.CELEvalError.__rsub__",false]],"__rtruediv__() (celpy.celtypes.doubletype method)":[[0,"celpy.celtypes.DoubleType.__rtruediv__",false]],"__rtruediv__() (celpy.celtypes.inttype method)":[[0,"celpy.celtypes.IntType.__rtruediv__",false]],"__rtruediv__() (celpy.celtypes.uinttype method)":[[0,"celpy.celtypes.UintType.__rtruediv__",false]],"__rtruediv__() (celpy.evaluation.celevalerror method)":[[0,"celpy.evaluation.CELEvalError.__rtruediv__",false]],"__str__() (celpy.celtypes.booltype method)":[[0,"celpy.celtypes.BoolType.__str__",false]],"__str__() (celpy.celtypes.doubletype method)":[[0,"celpy.celtypes.DoubleType.__str__",false]],"__str__() (celpy.celtypes.durationtype method)":[[0,"celpy.celtypes.DurationType.__str__",false]],"__str__() (celpy.celtypes.inttype method)":[[0,"celpy.celtypes.IntType.__str__",false]],"__str__() (celpy.celtypes.timestamptype method)":[[0,"celpy.celtypes.TimestampType.__str__",false]],"__str__() (celpy.celtypes.uinttype method)":[[0,"celpy.celtypes.UintType.__str__",false]],"__sub__() (celpy.celtypes.inttype method)":[[0,"celpy.celtypes.IntType.__sub__",false]],"__sub__() (celpy.celtypes.timestamptype method)":[[0,"celpy.celtypes.TimestampType.__sub__",false]],"__sub__() (celpy.celtypes.uinttype method)":[[0,"celpy.celtypes.UintType.__sub__",false]],"__sub__() (celpy.evaluation.celevalerror method)":[[0,"celpy.evaluation.CELEvalError.__sub__",false]],"__truediv__() (celpy.celtypes.doubletype method)":[[0,"celpy.celtypes.DoubleType.__truediv__",false]],"__truediv__() (celpy.celtypes.inttype method)":[[0,"celpy.celtypes.IntType.__truediv__",false]],"__truediv__() (celpy.celtypes.uinttype method)":[[0,"celpy.celtypes.UintType.__truediv__",false]],"__truediv__() (celpy.evaluation.celevalerror method)":[[0,"celpy.evaluation.CELEvalError.__truediv__",false]],"activation (class in celpy.evaluation)":[[0,"celpy.evaluation.Activation",false]],"activation() (celpy.__init__.environment method)":[[0,"celpy.__init__.Environment.activation",false]],"addition() (celpy.celparser.dumpast method)":[[0,"celpy.celparser.DumpAST.addition",false]],"addition() (celpy.evaluation.evaluator method)":[[0,"celpy.evaluation.Evaluator.addition",false]],"addition_add() (celpy.celparser.dumpast method)":[[0,"celpy.celparser.DumpAST.addition_add",false]],"addition_sub() (celpy.celparser.dumpast method)":[[0,"celpy.celparser.DumpAST.addition_sub",false]],"ambiguous_literals() (celpy.celparser.celparser static method)":[[0,"celpy.celparser.CELParser.ambiguous_literals",false]],"arg_type_value() (in module celpy.__main__)":[[0,"celpy.__main__.arg_type_value",false]],"boolean() (in module celpy.evaluation)":[[0,"celpy.evaluation.boolean",false]],"booltype (class in celpy.celtypes)":[[0,"celpy.celtypes.BoolType",false]],"build_macro_eval() (celpy.evaluation.evaluator method)":[[0,"celpy.evaluation.Evaluator.build_macro_eval",false]],"build_reduce_macro_eval() (celpy.evaluation.evaluator method)":[[0,"celpy.evaluation.Evaluator.build_reduce_macro_eval",false]],"build_ss_macro_eval() (celpy.evaluation.evaluator method)":[[0,"celpy.evaluation.Evaluator.build_ss_macro_eval",false]],"bytestype (class in celpy.celtypes)":[[0,"celpy.celtypes.BytesType",false]],"cel_eval() (celpy.__main__.cel_repl method)":[[0,"celpy.__main__.CEL_REPL.cel_eval",false]],"cel_parser (celpy.celparser.celparser attribute)":[[0,"celpy.celparser.CELParser.CEL_PARSER",false]],"cel_repl (class in celpy.__main__)":[[0,"celpy.__main__.CEL_REPL",false]],"celbytes() (in module celpy.evaluation)":[[0,"celpy.evaluation.celbytes",false]],"celevalerror":[[0,"celpy.evaluation.CELEvalError",false]],"celparseerror":[[0,"celpy.celparser.CELParseError",false]],"celparser (class in celpy.celparser)":[[0,"celpy.celparser.CELParser",false]],"celpy.__init__":[[0,"module-celpy.__init__",false]],"celpy.__main__":[[0,"module-celpy.__main__",false]],"celpy.celparser":[[0,"module-celpy.celparser",false]],"celpy.celtypes":[[0,"module-celpy.celtypes",false]],"celpy.evaluation":[[0,"module-celpy.evaluation",false]],"celstr() (in module celpy.evaluation)":[[0,"celpy.evaluation.celstr",false]],"celsyntaxerror":[[0,"celpy.evaluation.CELSyntaxError",false]],"celunsupportederror":[[0,"celpy.evaluation.CELUnsupportedError",false]],"clone() (celpy.evaluation.activation method)":[[0,"celpy.evaluation.Activation.clone",false]],"clone() (celpy.evaluation.namecontainer method)":[[0,"celpy.evaluation.NameContainer.clone",false]],"clone() (celpy.evaluation.referent method)":[[0,"celpy.evaluation.Referent.clone",false]],"compile() (celpy.__init__.environment method)":[[0,"celpy.__init__.Environment.compile",false]],"compiledrunner (class in celpy.__init__)":[[0,"celpy.__init__.CompiledRunner",false]],"conditionaland() (celpy.celparser.dumpast method)":[[0,"celpy.celparser.DumpAST.conditionaland",false]],"conditionaland() (celpy.evaluation.evaluator method)":[[0,"celpy.evaluation.Evaluator.conditionaland",false]],"conditionalor() (celpy.celparser.dumpast method)":[[0,"celpy.celparser.DumpAST.conditionalor",false]],"conditionalor() (celpy.evaluation.evaluator method)":[[0,"celpy.evaluation.Evaluator.conditionalor",false]],"default() (celpy.__main__.cel_repl method)":[[0,"celpy.__main__.CEL_REPL.default",false]],"dict_find_name() (celpy.evaluation.namecontainer static method)":[[0,"celpy.evaluation.NameContainer.dict_find_name",false]],"display() (celpy.celparser.dumpast class method)":[[0,"celpy.celparser.DumpAST.display",false]],"do_bye() (celpy.__main__.cel_repl method)":[[0,"celpy.__main__.CEL_REPL.do_bye",false]],"do_exit() (celpy.__main__.cel_repl method)":[[0,"celpy.__main__.CEL_REPL.do_exit",false]],"do_quit() (celpy.__main__.cel_repl method)":[[0,"celpy.__main__.CEL_REPL.do_quit",false]],"do_set() (celpy.__main__.cel_repl method)":[[0,"celpy.__main__.CEL_REPL.do_set",false]],"do_show() (celpy.__main__.cel_repl method)":[[0,"celpy.__main__.CEL_REPL.do_show",false]],"dot_ident() (celpy.celparser.dumpast method)":[[0,"celpy.celparser.DumpAST.dot_ident",false]],"dot_ident_arg() (celpy.celparser.dumpast method)":[[0,"celpy.celparser.DumpAST.dot_ident_arg",false]],"doubletype (class in celpy.celtypes)":[[0,"celpy.celtypes.DoubleType",false]],"dumpast (class in celpy.celparser)":[[0,"celpy.celparser.DumpAST",false]],"durationtype (class in celpy.celtypes)":[[0,"celpy.celtypes.DurationType",false]],"environment (class in celpy.__init__)":[[0,"celpy.__init__.Environment",false]],"error_text() (celpy.celparser.celparser method)":[[0,"celpy.celparser.CELParser.error_text",false]],"eval_error() (in module celpy.evaluation)":[[0,"celpy.evaluation.eval_error",false]],"evaluate() (celpy.__init__.compiledrunner method)":[[0,"celpy.__init__.CompiledRunner.evaluate",false]],"evaluate() (celpy.__init__.interpretedrunner method)":[[0,"celpy.__init__.InterpretedRunner.evaluate",false]],"evaluate() (celpy.__init__.runner method)":[[0,"celpy.__init__.Runner.evaluate",false]],"evaluate() (celpy.evaluation.evaluator method)":[[0,"celpy.evaluation.Evaluator.evaluate",false]],"evaluator (class in celpy.evaluation)":[[0,"celpy.evaluation.Evaluator",false]],"expr() (celpy.celparser.dumpast method)":[[0,"celpy.celparser.DumpAST.expr",false]],"expr() (celpy.evaluation.evaluator method)":[[0,"celpy.evaluation.Evaluator.expr",false]],"exprlist() (celpy.celparser.dumpast method)":[[0,"celpy.celparser.DumpAST.exprlist",false]],"exprlist() (celpy.evaluation.evaluator method)":[[0,"celpy.evaluation.Evaluator.exprlist",false]],"extended_name_path (celpy.evaluation.namecontainer attribute)":[[0,"celpy.evaluation.NameContainer.extended_name_path",false]],"fieldinits() (celpy.celparser.dumpast method)":[[0,"celpy.celparser.DumpAST.fieldinits",false]],"fieldinits() (celpy.evaluation.evaluator method)":[[0,"celpy.evaluation.Evaluator.fieldinits",false]],"find_name() (celpy.evaluation.namecontainer method)":[[0,"celpy.evaluation.NameContainer.find_name",false]],"findident (class in celpy.evaluation)":[[0,"celpy.evaluation.FindIdent",false]],"function_eval() (celpy.evaluation.evaluator method)":[[0,"celpy.evaluation.Evaluator.function_eval",false]],"function_matches() (in module celpy.evaluation)":[[0,"celpy.evaluation.function_matches",false]],"function_size() (in module celpy.evaluation)":[[0,"celpy.evaluation.function_size",false]],"functiontype (class in celpy.celtypes)":[[0,"celpy.celtypes.FunctionType",false]],"get_options() (in module celpy.__main__)":[[0,"celpy.__main__.get_options",false]],"getdate() (celpy.celtypes.timestamptype method)":[[0,"celpy.celtypes.TimestampType.getDate",false]],"getdayofmonth() (celpy.celtypes.timestamptype method)":[[0,"celpy.celtypes.TimestampType.getDayOfMonth",false]],"getdayofweek() (celpy.celtypes.timestamptype method)":[[0,"celpy.celtypes.TimestampType.getDayOfWeek",false]],"getdayofyear() (celpy.celtypes.timestamptype method)":[[0,"celpy.celtypes.TimestampType.getDayOfYear",false]],"getfullyear() (celpy.celtypes.timestamptype method)":[[0,"celpy.celtypes.TimestampType.getFullYear",false]],"gethours() (celpy.celtypes.durationtype method)":[[0,"celpy.celtypes.DurationType.getHours",false]],"gethours() (celpy.celtypes.timestamptype method)":[[0,"celpy.celtypes.TimestampType.getHours",false]],"getmilliseconds() (celpy.celtypes.durationtype method)":[[0,"celpy.celtypes.DurationType.getMilliseconds",false]],"getmilliseconds() (celpy.celtypes.timestamptype method)":[[0,"celpy.celtypes.TimestampType.getMilliseconds",false]],"getminutes() (celpy.celtypes.durationtype method)":[[0,"celpy.celtypes.DurationType.getMinutes",false]],"getminutes() (celpy.celtypes.timestamptype method)":[[0,"celpy.celtypes.TimestampType.getMinutes",false]],"getmonth() (celpy.celtypes.timestamptype method)":[[0,"celpy.celtypes.TimestampType.getMonth",false]],"getseconds() (celpy.celtypes.durationtype method)":[[0,"celpy.celtypes.DurationType.getSeconds",false]],"getseconds() (celpy.celtypes.timestamptype method)":[[0,"celpy.celtypes.TimestampType.getSeconds",false]],"ident() (celpy.celparser.dumpast method)":[[0,"celpy.celparser.DumpAST.ident",false]],"ident() (celpy.evaluation.findident method)":[[0,"celpy.evaluation.FindIdent.ident",false]],"ident_arg() (celpy.celparser.dumpast method)":[[0,"celpy.celparser.DumpAST.ident_arg",false]],"ident_pat (celpy.evaluation.namecontainer attribute)":[[0,"celpy.evaluation.NameContainer.ident_pat",false]],"ident_value() (celpy.evaluation.evaluator method)":[[0,"celpy.evaluation.Evaluator.ident_value",false]],"in_tree() (celpy.evaluation.findident class method)":[[0,"celpy.evaluation.FindIdent.in_tree",false]],"int32value (class in celpy.__init__)":[[0,"celpy.__init__.Int32Value",false]],"int64() (in module celpy.celtypes)":[[0,"celpy.celtypes.int64",false]],"interpretedrunner (class in celpy.__init__)":[[0,"celpy.__init__.InterpretedRunner",false]],"intro (celpy.__main__.cel_repl attribute)":[[0,"celpy.__main__.CEL_REPL.intro",false]],"inttype (class in celpy.celtypes)":[[0,"celpy.celtypes.IntType",false]],"list_lit() (celpy.celparser.dumpast method)":[[0,"celpy.celparser.DumpAST.list_lit",false]],"listtype (class in celpy.celtypes)":[[0,"celpy.celtypes.ListType",false]],"literal() (celpy.celparser.dumpast method)":[[0,"celpy.celparser.DumpAST.literal",false]],"literal() (celpy.evaluation.evaluator method)":[[0,"celpy.evaluation.Evaluator.literal",false]],"load_annotations() (celpy.evaluation.namecontainer method)":[[0,"celpy.evaluation.NameContainer.load_annotations",false]],"load_values() (celpy.evaluation.namecontainer method)":[[0,"celpy.evaluation.NameContainer.load_values",false]],"logger (celpy.__main__.cel_repl attribute)":[[0,"celpy.__main__.CEL_REPL.logger",false]],"logger (celpy.evaluation.evaluator attribute)":[[0,"celpy.evaluation.Evaluator.logger",false]],"logger (celpy.evaluation.namecontainer attribute)":[[0,"celpy.evaluation.NameContainer.logger",false]],"logical_and() (in module celpy.celtypes)":[[0,"celpy.celtypes.logical_and",false]],"logical_condition() (in module celpy.celtypes)":[[0,"celpy.celtypes.logical_condition",false]],"logical_not() (in module celpy.celtypes)":[[0,"celpy.celtypes.logical_not",false]],"logical_or() (in module celpy.celtypes)":[[0,"celpy.celtypes.logical_or",false]],"macro_has_eval() (celpy.evaluation.evaluator method)":[[0,"celpy.evaluation.Evaluator.macro_has_eval",false]],"main() (in module celpy.__main__)":[[0,"celpy.__main__.main",false]],"map_lit() (celpy.celparser.dumpast method)":[[0,"celpy.celparser.DumpAST.map_lit",false]],"mapinits() (celpy.celparser.dumpast method)":[[0,"celpy.celparser.DumpAST.mapinits",false]],"mapinits() (celpy.evaluation.evaluator method)":[[0,"celpy.evaluation.Evaluator.mapinits",false]],"maptype (class in celpy.celtypes)":[[0,"celpy.celtypes.MapType",false]],"maxseconds (celpy.celtypes.durationtype attribute)":[[0,"celpy.celtypes.DurationType.MaxSeconds",false]],"member() (celpy.evaluation.evaluator method)":[[0,"celpy.evaluation.Evaluator.member",false]],"member_dot() (celpy.celparser.dumpast method)":[[0,"celpy.celparser.DumpAST.member_dot",false]],"member_dot() (celpy.evaluation.evaluator method)":[[0,"celpy.evaluation.Evaluator.member_dot",false]],"member_dot_arg() (celpy.celparser.dumpast method)":[[0,"celpy.celparser.DumpAST.member_dot_arg",false]],"member_dot_arg() (celpy.evaluation.evaluator method)":[[0,"celpy.evaluation.Evaluator.member_dot_arg",false]],"member_index() (celpy.celparser.dumpast method)":[[0,"celpy.celparser.DumpAST.member_index",false]],"member_index() (celpy.evaluation.evaluator method)":[[0,"celpy.evaluation.Evaluator.member_index",false]],"member_object() (celpy.celparser.dumpast method)":[[0,"celpy.celparser.DumpAST.member_object",false]],"member_object() (celpy.evaluation.evaluator method)":[[0,"celpy.evaluation.Evaluator.member_object",false]],"messagetype (class in celpy.celtypes)":[[0,"celpy.celtypes.MessageType",false]],"method_eval() (celpy.evaluation.evaluator method)":[[0,"celpy.evaluation.Evaluator.method_eval",false]],"minseconds (celpy.celtypes.durationtype attribute)":[[0,"celpy.celtypes.DurationType.MinSeconds",false]],"module":[[0,"module-celpy.__init__",false],[0,"module-celpy.__main__",false],[0,"module-celpy.celparser",false],[0,"module-celpy.celtypes",false],[0,"module-celpy.evaluation",false]],"multiplication() (celpy.celparser.dumpast method)":[[0,"celpy.celparser.DumpAST.multiplication",false]],"multiplication() (celpy.evaluation.evaluator method)":[[0,"celpy.evaluation.Evaluator.multiplication",false]],"multiplication_div() (celpy.celparser.dumpast method)":[[0,"celpy.celparser.DumpAST.multiplication_div",false]],"multiplication_mod() (celpy.celparser.dumpast method)":[[0,"celpy.celparser.DumpAST.multiplication_mod",false]],"multiplication_mul() (celpy.celparser.dumpast method)":[[0,"celpy.celparser.DumpAST.multiplication_mul",false]],"namecontainer (class in celpy.evaluation)":[[0,"celpy.evaluation.NameContainer",false]],"namecontainer.notfound":[[0,"celpy.evaluation.NameContainer.NotFound",false]],"nanosecondspersecond (celpy.celtypes.durationtype attribute)":[[0,"celpy.celtypes.DurationType.NanosecondsPerSecond",false]],"nested_activation() (celpy.evaluation.activation method)":[[0,"celpy.evaluation.Activation.nested_activation",false]],"new_activation() (celpy.__init__.runner method)":[[0,"celpy.__init__.Runner.new_activation",false]],"nulltype (class in celpy.celtypes)":[[0,"celpy.celtypes.NullType",false]],"operator_in() (in module celpy.evaluation)":[[0,"celpy.evaluation.operator_in",false]],"packagetype (class in celpy.celtypes)":[[0,"celpy.celtypes.PackageType",false]],"paren_expr() (celpy.celparser.dumpast method)":[[0,"celpy.celparser.DumpAST.paren_expr",false]],"parent_iter() (celpy.evaluation.namecontainer method)":[[0,"celpy.evaluation.NameContainer.parent_iter",false]],"parse() (celpy.celparser.celparser method)":[[0,"celpy.celparser.CELParser.parse",false]],"preloop() (celpy.__main__.cel_repl method)":[[0,"celpy.__main__.CEL_REPL.preloop",false]],"primary() (celpy.evaluation.evaluator method)":[[0,"celpy.evaluation.Evaluator.primary",false]],"process_json_doc() (in module celpy.__main__)":[[0,"celpy.__main__.process_json_doc",false]],"program() (celpy.__init__.environment method)":[[0,"celpy.__init__.Environment.program",false]],"prompt (celpy.__main__.cel_repl attribute)":[[0,"celpy.__main__.CEL_REPL.prompt",false]],"referent (class in celpy.evaluation)":[[0,"celpy.evaluation.Referent",false]],"relation() (celpy.celparser.dumpast method)":[[0,"celpy.celparser.DumpAST.relation",false]],"relation() (celpy.evaluation.evaluator method)":[[0,"celpy.evaluation.Evaluator.relation",false]],"relation_eq() (celpy.celparser.dumpast method)":[[0,"celpy.celparser.DumpAST.relation_eq",false]],"relation_ge() (celpy.celparser.dumpast method)":[[0,"celpy.celparser.DumpAST.relation_ge",false]],"relation_gt() (celpy.celparser.dumpast method)":[[0,"celpy.celparser.DumpAST.relation_gt",false]],"relation_in() (celpy.celparser.dumpast method)":[[0,"celpy.celparser.DumpAST.relation_in",false]],"relation_le() (celpy.celparser.dumpast method)":[[0,"celpy.celparser.DumpAST.relation_le",false]],"relation_lt() (celpy.celparser.dumpast method)":[[0,"celpy.celparser.DumpAST.relation_lt",false]],"relation_ne() (celpy.celparser.dumpast method)":[[0,"celpy.celparser.DumpAST.relation_ne",false]],"resolve_name() (celpy.evaluation.namecontainer method)":[[0,"celpy.evaluation.NameContainer.resolve_name",false]],"resolve_variable() (celpy.evaluation.activation method)":[[0,"celpy.evaluation.Activation.resolve_variable",false]],"runner (class in celpy.__init__)":[[0,"celpy.__init__.Runner",false]],"scale (celpy.celtypes.durationtype attribute)":[[0,"celpy.celtypes.DurationType.scale",false]],"set_activation() (celpy.evaluation.evaluator method)":[[0,"celpy.evaluation.Evaluator.set_activation",false]],"stringtype (class in celpy.celtypes)":[[0,"celpy.celtypes.StringType",false]],"sub_evaluator() (celpy.evaluation.evaluator method)":[[0,"celpy.evaluation.Evaluator.sub_evaluator",false]],"timestamptype (class in celpy.celtypes)":[[0,"celpy.celtypes.TimestampType",false]],"trace() (in module celpy.evaluation)":[[0,"celpy.evaluation.trace",false]],"tree_dump() (in module celpy.celparser)":[[0,"celpy.celparser.tree_dump",false]],"type_matched() (in module celpy.celtypes)":[[0,"celpy.celtypes.type_matched",false]],"type_name_mapping (celpy.celtypes.typetype attribute)":[[0,"celpy.celtypes.TypeType.type_name_mapping",false]],"typetype (class in celpy.celtypes)":[[0,"celpy.celtypes.TypeType",false]],"tz_aliases (celpy.celtypes.timestamptype attribute)":[[0,"celpy.celtypes.TimestampType.TZ_ALIASES",false]],"tz_name_lookup() (celpy.celtypes.timestamptype class method)":[[0,"celpy.celtypes.TimestampType.tz_name_lookup",false]],"tz_offset_parse() (celpy.celtypes.timestamptype class method)":[[0,"celpy.celtypes.TimestampType.tz_offset_parse",false]],"tz_parse() (celpy.celtypes.timestamptype static method)":[[0,"celpy.celtypes.TimestampType.tz_parse",false]],"uint64() (in module celpy.celtypes)":[[0,"celpy.celtypes.uint64",false]],"uinttype (class in celpy.celtypes)":[[0,"celpy.celtypes.UintType",false]],"unary() (celpy.celparser.dumpast method)":[[0,"celpy.celparser.DumpAST.unary",false]],"unary() (celpy.evaluation.evaluator method)":[[0,"celpy.evaluation.Evaluator.unary",false]],"unary_neg() (celpy.celparser.dumpast method)":[[0,"celpy.celparser.DumpAST.unary_neg",false]],"unary_not() (celpy.celparser.dumpast method)":[[0,"celpy.celparser.DumpAST.unary_not",false]],"valid_key_type() (celpy.celtypes.maptype static method)":[[0,"celpy.celtypes.MapType.valid_key_type",false]],"value (celpy.evaluation.referent property)":[[0,"celpy.evaluation.Referent.value",false]],"visit_children() (celpy.evaluation.evaluator method)":[[0,"celpy.evaluation.Evaluator.visit_children",false]],"with_traceback() (celpy.evaluation.celevalerror method)":[[0,"celpy.evaluation.CELEvalError.with_traceback",false]]},"objects":{"celpy":[[0,0,0,"-","__init__"],[0,0,0,"-","__main__"],[0,0,0,"-","celparser"],[0,0,0,"-","celtypes"],[0,0,0,"-","evaluation"]],"celpy.__init__":[[0,1,1,"","CompiledRunner"],[0,1,1,"","Environment"],[0,1,1,"","Int32Value"],[0,1,1,"","InterpretedRunner"],[0,1,1,"","Runner"]],"celpy.__init__.CompiledRunner":[[0,2,1,"","__init__"],[0,2,1,"","evaluate"]],"celpy.__init__.Environment":[[0,2,1,"","__init__"],[0,2,1,"","activation"],[0,2,1,"","compile"],[0,2,1,"","program"]],"celpy.__init__.Int32Value":[[0,2,1,"","__new__"]],"celpy.__init__.InterpretedRunner":[[0,2,1,"","evaluate"]],"celpy.__init__.Runner":[[0,2,1,"","__init__"],[0,2,1,"","evaluate"],[0,2,1,"","new_activation"]],"celpy.__main__":[[0,1,1,"","CEL_REPL"],[0,4,1,"","arg_type_value"],[0,4,1,"","get_options"],[0,4,1,"","main"],[0,4,1,"","process_json_doc"]],"celpy.__main__.CEL_REPL":[[0,2,1,"","cel_eval"],[0,2,1,"","default"],[0,2,1,"","do_bye"],[0,2,1,"","do_exit"],[0,2,1,"","do_quit"],[0,2,1,"","do_set"],[0,2,1,"","do_show"],[0,3,1,"","intro"],[0,3,1,"","logger"],[0,2,1,"","preloop"],[0,3,1,"","prompt"]],"celpy.celparser":[[0,5,1,"","CELParseError"],[0,1,1,"","CELParser"],[0,1,1,"","DumpAST"],[0,4,1,"","tree_dump"]],"celpy.celparser.CELParseError":[[0,2,1,"","__init__"]],"celpy.celparser.CELParser":[[0,3,1,"","CEL_PARSER"],[0,2,1,"","__init__"],[0,2,1,"","ambiguous_literals"],[0,2,1,"","error_text"],[0,2,1,"","parse"]],"celpy.celparser.DumpAST":[[0,2,1,"","__init__"],[0,3,1,"","__parameters__"],[0,2,1,"","addition"],[0,2,1,"","addition_add"],[0,2,1,"","addition_sub"],[0,2,1,"","conditionaland"],[0,2,1,"","conditionalor"],[0,2,1,"","display"],[0,2,1,"","dot_ident"],[0,2,1,"","dot_ident_arg"],[0,2,1,"","expr"],[0,2,1,"","exprlist"],[0,2,1,"","fieldinits"],[0,2,1,"","ident"],[0,2,1,"","ident_arg"],[0,2,1,"","list_lit"],[0,2,1,"","literal"],[0,2,1,"","map_lit"],[0,2,1,"","mapinits"],[0,2,1,"","member_dot"],[0,2,1,"","member_dot_arg"],[0,2,1,"","member_index"],[0,2,1,"","member_object"],[0,2,1,"","multiplication"],[0,2,1,"","multiplication_div"],[0,2,1,"","multiplication_mod"],[0,2,1,"","multiplication_mul"],[0,2,1,"","paren_expr"],[0,2,1,"","relation"],[0,2,1,"","relation_eq"],[0,2,1,"","relation_ge"],[0,2,1,"","relation_gt"],[0,2,1,"","relation_in"],[0,2,1,"","relation_le"],[0,2,1,"","relation_lt"],[0,2,1,"","relation_ne"],[0,2,1,"","unary"],[0,2,1,"","unary_neg"],[0,2,1,"","unary_not"]],"celpy.celtypes":[[0,1,1,"","BoolType"],[0,1,1,"","BytesType"],[0,1,1,"","DoubleType"],[0,1,1,"","DurationType"],[0,1,1,"","FunctionType"],[0,1,1,"","IntType"],[0,1,1,"","ListType"],[0,1,1,"","MapType"],[0,1,1,"","MessageType"],[0,1,1,"","NullType"],[0,1,1,"","PackageType"],[0,1,1,"","StringType"],[0,1,1,"","TimestampType"],[0,1,1,"","TypeType"],[0,1,1,"","UintType"],[0,4,1,"","int64"],[0,4,1,"","logical_and"],[0,4,1,"","logical_condition"],[0,4,1,"","logical_not"],[0,4,1,"","logical_or"],[0,4,1,"","type_matched"],[0,4,1,"","uint64"]],"celpy.celtypes.BoolType":[[0,2,1,"","__hash__"],[0,2,1,"","__neg__"],[0,2,1,"","__new__"],[0,2,1,"","__repr__"],[0,2,1,"","__str__"]],"celpy.celtypes.BytesType":[[0,2,1,"","__new__"],[0,2,1,"","__repr__"]],"celpy.celtypes.DoubleType":[[0,2,1,"","__eq__"],[0,2,1,"","__hash__"],[0,2,1,"","__mod__"],[0,2,1,"","__ne__"],[0,2,1,"","__neg__"],[0,2,1,"","__new__"],[0,2,1,"","__repr__"],[0,2,1,"","__rmod__"],[0,2,1,"","__rtruediv__"],[0,2,1,"","__str__"],[0,2,1,"","__truediv__"]],"celpy.celtypes.DurationType":[[0,3,1,"","MaxSeconds"],[0,3,1,"","MinSeconds"],[0,3,1,"","NanosecondsPerSecond"],[0,2,1,"","__add__"],[0,2,1,"","__new__"],[0,2,1,"","__radd__"],[0,2,1,"","__repr__"],[0,2,1,"","__str__"],[0,2,1,"","getHours"],[0,2,1,"","getMilliseconds"],[0,2,1,"","getMinutes"],[0,2,1,"","getSeconds"],[0,3,1,"","scale"]],"celpy.celtypes.FunctionType":[[0,2,1,"","__call__"]],"celpy.celtypes.IntType":[[0,2,1,"","__add__"],[0,2,1,"","__eq__"],[0,2,1,"","__floordiv__"],[0,2,1,"","__ge__"],[0,2,1,"","__gt__"],[0,2,1,"","__hash__"],[0,2,1,"","__le__"],[0,2,1,"","__lt__"],[0,2,1,"","__mod__"],[0,2,1,"","__mul__"],[0,2,1,"","__ne__"],[0,2,1,"","__neg__"],[0,2,1,"","__new__"],[0,2,1,"","__radd__"],[0,2,1,"","__repr__"],[0,2,1,"","__rfloordiv__"],[0,2,1,"","__rmod__"],[0,2,1,"","__rmul__"],[0,2,1,"","__rsub__"],[0,2,1,"","__rtruediv__"],[0,2,1,"","__str__"],[0,2,1,"","__sub__"],[0,2,1,"","__truediv__"]],"celpy.celtypes.ListType":[[0,2,1,"","__eq__"],[0,2,1,"","__ge__"],[0,2,1,"","__gt__"],[0,3,1,"","__hash__"],[0,2,1,"","__le__"],[0,2,1,"","__lt__"],[0,2,1,"","__ne__"],[0,3,1,"","__orig_bases__"],[0,3,1,"","__parameters__"],[0,2,1,"","__repr__"]],"celpy.celtypes.MapType":[[0,2,1,"","__eq__"],[0,2,1,"","__getitem__"],[0,3,1,"","__hash__"],[0,2,1,"","__init__"],[0,2,1,"","__ne__"],[0,3,1,"","__orig_bases__"],[0,3,1,"","__parameters__"],[0,2,1,"","__repr__"],[0,2,1,"","valid_key_type"]],"celpy.celtypes.MessageType":[[0,2,1,"","__init__"],[0,3,1,"","__parameters__"]],"celpy.celtypes.NullType":[[0,2,1,"","__eq__"],[0,3,1,"","__hash__"],[0,2,1,"","__ne__"]],"celpy.celtypes.PackageType":[[0,3,1,"","__parameters__"]],"celpy.celtypes.StringType":[[0,2,1,"","__eq__"],[0,2,1,"","__hash__"],[0,2,1,"","__ne__"],[0,2,1,"","__new__"],[0,2,1,"","__repr__"]],"celpy.celtypes.TimestampType":[[0,3,1,"","TZ_ALIASES"],[0,2,1,"","__add__"],[0,2,1,"","__new__"],[0,2,1,"","__radd__"],[0,2,1,"","__repr__"],[0,2,1,"","__str__"],[0,2,1,"","__sub__"],[0,2,1,"","getDate"],[0,2,1,"","getDayOfMonth"],[0,2,1,"","getDayOfWeek"],[0,2,1,"","getDayOfYear"],[0,2,1,"","getFullYear"],[0,2,1,"","getHours"],[0,2,1,"","getMilliseconds"],[0,2,1,"","getMinutes"],[0,2,1,"","getMonth"],[0,2,1,"","getSeconds"],[0,2,1,"","tz_name_lookup"],[0,2,1,"","tz_offset_parse"],[0,2,1,"","tz_parse"]],"celpy.celtypes.TypeType":[[0,2,1,"","__eq__"],[0,3,1,"","__hash__"],[0,2,1,"","__init__"],[0,3,1,"","type_name_mapping"]],"celpy.celtypes.UintType":[[0,2,1,"","__add__"],[0,2,1,"","__eq__"],[0,2,1,"","__floordiv__"],[0,2,1,"","__hash__"],[0,2,1,"","__mod__"],[0,2,1,"","__mul__"],[0,2,1,"","__ne__"],[0,2,1,"","__neg__"],[0,2,1,"","__new__"],[0,2,1,"","__radd__"],[0,2,1,"","__repr__"],[0,2,1,"","__rfloordiv__"],[0,2,1,"","__rmod__"],[0,2,1,"","__rmul__"],[0,2,1,"","__rsub__"],[0,2,1,"","__rtruediv__"],[0,2,1,"","__str__"],[0,2,1,"","__sub__"],[0,2,1,"","__truediv__"]],"celpy.evaluation":[[0,1,1,"","Activation"],[0,5,1,"","CELEvalError"],[0,5,1,"","CELSyntaxError"],[0,5,1,"","CELUnsupportedError"],[0,1,1,"","Evaluator"],[0,1,1,"","FindIdent"],[0,1,1,"","NameContainer"],[0,1,1,"","Referent"],[0,4,1,"","boolean"],[0,4,1,"","celbytes"],[0,4,1,"","celstr"],[0,4,1,"","eval_error"],[0,4,1,"","function_matches"],[0,4,1,"","function_size"],[0,4,1,"","operator_in"],[0,4,1,"","trace"]],"celpy.evaluation.Activation":[[0,2,1,"","__init__"],[0,2,1,"","__repr__"],[0,2,1,"","clone"],[0,2,1,"","nested_activation"],[0,2,1,"","resolve_variable"]],"celpy.evaluation.CELEvalError":[[0,2,1,"","__add__"],[0,2,1,"","__call__"],[0,2,1,"","__eq__"],[0,2,1,"","__floordiv__"],[0,3,1,"","__hash__"],[0,2,1,"","__init__"],[0,2,1,"","__mod__"],[0,2,1,"","__mul__"],[0,2,1,"","__neg__"],[0,2,1,"","__pow__"],[0,2,1,"","__radd__"],[0,2,1,"","__repr__"],[0,2,1,"","__rfloordiv__"],[0,2,1,"","__rmod__"],[0,2,1,"","__rmul__"],[0,2,1,"","__rpow__"],[0,2,1,"","__rsub__"],[0,2,1,"","__rtruediv__"],[0,2,1,"","__sub__"],[0,2,1,"","__truediv__"],[0,2,1,"","with_traceback"]],"celpy.evaluation.CELSyntaxError":[[0,2,1,"","__init__"]],"celpy.evaluation.CELUnsupportedError":[[0,2,1,"","__init__"]],"celpy.evaluation.Evaluator":[[0,3,1,"","__abstractmethods__"],[0,2,1,"","__init__"],[0,3,1,"","__parameters__"],[0,2,1,"","addition"],[0,2,1,"","build_macro_eval"],[0,2,1,"","build_reduce_macro_eval"],[0,2,1,"","build_ss_macro_eval"],[0,2,1,"","conditionaland"],[0,2,1,"","conditionalor"],[0,2,1,"","evaluate"],[0,2,1,"","expr"],[0,2,1,"","exprlist"],[0,2,1,"","fieldinits"],[0,2,1,"","function_eval"],[0,2,1,"","ident_value"],[0,2,1,"","literal"],[0,3,1,"","logger"],[0,2,1,"","macro_has_eval"],[0,2,1,"","mapinits"],[0,2,1,"","member"],[0,2,1,"","member_dot"],[0,2,1,"","member_dot_arg"],[0,2,1,"","member_index"],[0,2,1,"","member_object"],[0,2,1,"","method_eval"],[0,2,1,"","multiplication"],[0,2,1,"","primary"],[0,2,1,"","relation"],[0,2,1,"","set_activation"],[0,2,1,"","sub_evaluator"],[0,2,1,"","unary"],[0,2,1,"","visit_children"]],"celpy.evaluation.FindIdent":[[0,2,1,"","__init__"],[0,3,1,"","__parameters__"],[0,2,1,"","ident"],[0,2,1,"","in_tree"]],"celpy.evaluation.NameContainer":[[0,5,1,"","NotFound"],[0,2,1,"","__init__"],[0,3,1,"","__orig_bases__"],[0,3,1,"","__parameters__"],[0,2,1,"","__repr__"],[0,2,1,"","clone"],[0,2,1,"","dict_find_name"],[0,3,1,"","extended_name_path"],[0,2,1,"","find_name"],[0,3,1,"","ident_pat"],[0,2,1,"","load_annotations"],[0,2,1,"","load_values"],[0,3,1,"","logger"],[0,2,1,"","parent_iter"],[0,2,1,"","resolve_name"]],"celpy.evaluation.Referent":[[0,2,1,"","__init__"],[0,2,1,"","__repr__"],[0,2,1,"","clone"],[0,6,1,"","value"]]},"objnames":{"0":["py","module","Python module"],"1":["py","class","Python class"],"2":["py","method","Python method"],"3":["py","attribute","Python attribute"],"4":["py","function","Python function"],"5":["py","exception","Python exception"],"6":["py","property","Python property"]},"objtypes":{"0":"py:module","1":"py:class","2":"py:method","3":"py:attribute","4":"py:function","5":"py:exception","6":"py:property"},"terms":{"":[0,1,2,3,6],"0":[0,1,2,6],"00":6,"001":0,"002":1,"01":1,"011":1,"01ab23cd":1,"02":[0,1],"03":1,"03t16":6,"03z":6,"04":[1,6],"05":1,"06":0,"06t05":6,"07":[1,6],"08":[1,6],"084":1,"09":0,"1":[0,1,2,3,6],"10":[0,1],"100":[1,2],"1000":1,"10000":1,"1000000000":0,"1004":1,"101":1,"1021":1,"1023":1,"10240":1,"1026":1,"103":1,"1034":1,"104":1,"1041":1,"1045":1,"105":1,"1057":1,"106":1,"1066":1,"108":1,"1080":1,"10am":1,"11":1,"111":1,"1124":1,"113":[0,2],"1144":1,"115":1,"116":1,"117":1,"1178":1,"1180":1,"1191":1,"12":1,"121":1,"1211":1,"122":1,"123":[0,1,6],"1232":1,"1234567890":0,"123u":0,"125":1,"127":[0,1],"128":1,"1295":1,"12ab34cd":1,"13":[2,5],"131":1,"132":1,"1333":1,"134":1,"136":1,"138":0,"13t23":0,"14":1,"140":6,"1410":1,"14159":0,"1415929203539825":2,"1437":1,"145":1,"147":1,"1474":1,"149":0,"1493":1,"15":[1,6],"1505":1,"1513":1,"154":1,"1584":1,"16":1,"161":1,"1613":1,"162":1,"167":1,"1686":1,"17":1,"1704":1,"171":1,"1728":1,"173":1,"1779":1,"178":1,"1787":1,"179":1,"1793":1,"1799":1,"18":1,"182":1,"183":1,"184":1,"18446744073709551615":0,"185":1,"189":1,"19":1,"190":1,"194":1,"1954":1,"196":1,"197":1,"198":1,"1e":0,"1st":[1,6],"1u":0,"2":[0,1,6],"20":1,"2009":0,"2012":1,"2016":1,"2017":1,"2018":6,"2019":1,"2020":1,"2021":1,"203":1,"2048":1,"205":1,"206":1,"21":[1,6],"212":1,"21d":1,"22":1,"223":1,"2242":1,"228":1,"2292":1,"23":[1,6],"232":1,"237":1,"238":1,"24":1,"24h":6,"24x5":1,"25":1,"250":1,"2500000":1,"257":1,"26":1,"262":1,"268":1,"27":[1,6],"273":1,"277":[0,1],"2788":1,"28":1,"285":0,"29":1,"293":1,"299":1,"2h45m":0,"2nd":6,"3":[0,1,2,5],"30":[0,1],"300m":0,"301":1,"305":1,"30z":0,"31":[0,2],"313":1,"314":1,"315576000000":0,"317":1,"318":1,"32":1,"325":1,"329":1,"34":1,"341":1,"345":1,"348":1,"35":1,"355":[0,2],"35dai":1,"36":1,"3600":0,"364":1,"37":1,"372":1,"373":1,"390":1,"3a":1,"4":[0,1],"40":[1,6],"400":1,"402":1,"408":1,"409":1,"41":1,"412":1,"413":1,"42":[0,1,2,3],"424":1,"429":1,"42u":0,"43":1,"431":1,"432u":0,"436":1,"438":1,"443":1,"448":1,"45":1,"453":1,"455":1,"456":0,"47":[1,6],"48":1,"485":1,"490":1,"493":1,"5":[0,1],"50":1,"500":6,"5000":1,"505":1,"5103":1,"512":1,"5120":1,"514":1,"516":1,"52":1,"54":1,"548":1,"549":1,"55":1,"556":1,"562":1,"563":1,"567":1,"569":1,"57":1,"575":1,"5759":6,"58":1,"592":1,"597":1,"5h":0,"6":[0,1,3],"60":[0,1],"600":6,"604":1,"608":1,"610":1,"612000":1,"615":1,"619":1,"622":1,"628":1,"63":0,"64":[1,6,7],"643":1,"644":1,"65":1,"656":1,"659":1,"66":1,"662":1,"666":1,"667":1,"676":1,"69":1,"692":1,"6am":1,"6hr":1,"6pm":1,"7":[1,3],"711":1,"720":1,"729":1,"739":1,"74":1,"744":1,"748":1,"758":1,"767":1,"78":1,"786":1,"791":1,"7am":1,"7pm":1,"8":[1,6],"80":1,"806":1,"8098":1,"81":1,"812":1,"826":1,"829":1,"830985915492956":2,"842":1,"845":1,"86":1,"86400":[0,1],"87":1,"88":1,"8x5":1,"9":[0,1],"90":1,"92":1,"9223372036854775807":0,"9223372036854775808":0,"93":1,"932":1,"934":1,"94":1,"97":1,"98":1,"987":1,"988":1,"99":1,"9pm":1,"A":[0,1,6,7],"AND":0,"And":[0,2],"As":6,"At":0,"But":0,"By":0,"For":[0,1,2,5,6],"If":[0,1,3,6],"In":[0,1,3,5,6],"It":[0,1,2,6,7],"NOT":[0,1],"No":[0,1,6],"Not":[0,1],"Of":1,"On":[1,6],"One":[0,1],"Or":0,"That":0,"The":[0,1,2,3,4,5,7],"Then":[1,6,7],"There":[0,1,2,4,6,7],"These":[0,1,6],"To":[0,3,6],"With":[0,1],"_":[0,6],"__abstractmethods__":0,"__add__":0,"__call__":[0,6],"__eq__":0,"__floordiv__":0,"__ge__":0,"__getitem__":0,"__gt__":0,"__hash__":0,"__init__":[0,1,6],"__isinstance__":0,"__le__":0,"__lt__":0,"__main__":4,"__mod__":0,"__mul__":0,"__ne__":0,"__neg__":0,"__new__":0,"__orig_bases__":0,"__parameters__":0,"__pow__":0,"__radd__":0,"__repr__":0,"__rfloordiv__":0,"__rmod__":0,"__rmul__":0,"__rpow__":0,"__rsub__":0,"__rtruediv__":0,"__str__":0,"__sub__":0,"__traceback__":0,"__truediv__":0,"_a":0,"_pull_ami_snapshot":1,"_pull_asg_imag":1,"_pull_asg_snapshot":1,"_pull_ec2_imag":1,"ab":0,"abandond":1,"abc":1,"abil":2,"abl":[0,1],"about":[1,4,6],"abov":[0,1,6],"absenc":1,"absent":1,"absolut":0,"abstract":0,"accept":[0,1],"access":0,"access_kei":1,"accesskeysperuserquota":1,"accesslog":1,"account":6,"account_id":1,"account_shield_subscript":1,"accountaccesskeyspres":1,"accountcredentialreport":1,"accountmfaen":1,"accountnumb":1,"accountpasswordpolici":1,"accounts_url":1,"accountsigningcertificatespres":1,"achiev":0,"acl":1,"aclawss3cidr":1,"aclsubnetfilt":1,"acm":[1,6],"acquir":1,"across":[1,4],"act":1,"action":[1,6],"action_d":1,"activ":[0,1,6,7],"actual":0,"ad":[0,1],"adapt":[6,7],"add":[0,1,5,7],"addit":[0,1,6,7],"addition":[0,1,7],"addition_add":0,"addition_sub":0,"additionalproperti":1,"addr":1,"address":1,"adher":1,"admin":1,"administratoraccess":1,"advanc":1,"advantag":2,"advisor":1,"aes128":1,"aes256":1,"after":[0,1],"ag":6,"again":0,"against":1,"agefilt":1,"agre":0,"alarm":1,"alb":1,"alblog":1,"alert":1,"alia":0,"alias":[0,1],"aliasnam":1,"all":[0,6],"all_dbsubenet_group":1,"all_imag":1,"all_instance_profil":1,"all_launch_configuration_nam":1,"all_scan_group":1,"all_service_rol":1,"all_snapshot":1,"all_valu":1,"alloc":1,"allocatedstorag":1,"allow":[0,6],"allow_websit":1,"allowalliampolici":1,"alon":1,"along":[0,1],"alreadi":6,"also":[0,1,2,6,7],"altern":[0,1,6],"alwai":[0,1],"amazon":1,"amazonaw":1,"amazons3":1,"ambigu":0,"ambiguous_liter":0,"america":1,"amicrossaccountfilt":1,"among":1,"an":[0,1,6,7],"analysi":1,"analyt":1,"ancient":1,"ani":[0,1,6],"annot":[0,1,6],"anoth":[1,4],"anyof":1,"ap":1,"api":[1,4],"apigw":1,"apikeyrequir":1,"app":[1,7],"appear":[0,1,6],"appelb":1,"appelbdefaultvpcfilt":1,"appelbhealthcheckprotocolmismatchfilt":1,"appelblistenerfilt":1,"appelbmetr":1,"appelbtargetgroup":1,"appelbtargetgroupdefaultvpcfilt":1,"appelbtargetgroupfilt":1,"append":0,"appid":1,"appli":[0,1,6],"applic":[0,1,3,4],"applicationnam":1,"approach":[0,6],"approxim":0,"ar":[0,1,3,4,6,7],"architectur":5,"aren":[0,1],"arg":[0,3,6],"arg_name_arg":6,"arg_type_valu":0,"argtyp":6,"argument":[3,6],"argv":0,"arithmetic_oper":0,"aritmet":2,"arm64":5,"arn":1,"arn_split":1,"around":1,"arrai":[0,1],"arriv":[1,6],"asg":1,"assert":[1,6],"assign":[0,1],"assoc":1,"associ":[0,1],"associatepublicipaddress":1,"assum":[0,1],"assumerolepolicysizequota":1,"assumpt":1,"assur":0,"ast":[0,6,7],"ast2":0,"asv":1,"asvredact":1,"atm":1,"atom":6,"atot":2,"attach":[0,1],"attachedinstancefilt":1,"attachedpoliciespergroupquota":1,"attachedpoliciesperrolequota":1,"attachedpoliciesperuserquota":1,"attachedvolum":1,"attachmentcount":1,"attack":1,"attempt":[0,1],"attr":1,"attribit":1,"attribut":[0,6],"attributesfilt":1,"audit":1,"augment":1,"aurora":1,"auth":6,"author":6,"auto":1,"automat":1,"autosc":1,"autoscal":1,"avail":[0,3,6],"aval":1,"averag":1,"avoid":1,"aw":1,"aws_ebs_degraded_ebs_volume_perform":1,"aws_ebs_volume_lost":1,"awsec2":1,"awslambda":1,"ax":2,"az1":1,"azr":1,"azur":1,"b":[0,2,6],"ba":1,"back":[0,1],"backoff":1,"backup":1,"bad1":6,"bad2":6,"balanc":[1,6],"base":[0,1,4,6],"base_funct":0,"baseexcept":0,"baselin":1,"basi":1,"basic":1,"batch":1,"baz":0,"bc":0,"becaus":[0,1,6],"becom":[0,1],"been":[0,1,7],"befor":[0,1],"behav":0,"behavior":[0,1],"being":1,"below":0,"better":[0,1],"between":[0,1],"beyond":0,"big":6,"binari":[0,6],"bind":[0,3,4,7],"bit":[1,2,7],"blacklist":1,"blob":[0,1,6],"blockdevicemap":1,"bool":[0,4,6],"bool_valu":0,"boolean":[0,4,6],"boolean_to_statu":0,"booleangroupfilt":6,"booltyp":[0,6,7],"boolvalu":0,"both":[0,1],"boto3":1,"bottl":0,"bottom":0,"bound":0,"bracket":1,"breakdown":6,"bring":1,"broken":1,"broker":1,"bucketencrypt":1,"bucketnotificationfilt":1,"buffer":0,"build":[0,1,6,7],"build_macro_ev":0,"build_reduce_macro_ev":0,"build_ss_macro_ev":0,"buildsecuritygroupfilt":1,"buildsubnetfilt":1,"buildvpcfilt":1,"built":[0,1,2,6,7],"bulki":1,"bunch":1,"byol":1,"byt":0,"byte":[0,7],"bytes_valu":0,"bytestyp":[0,7],"bytesvalu":0,"c":[0,2,6],"c7n":[0,4],"c7n_interpreted_runn":1,"c7n_s3_policy_requir":1,"c7ncontext":1,"c7nlib":1,"cach":4,"calcul":2,"call":[0,1],"callabl":0,"can":[0,1,2,3,4,5,6,7],"cannot":[0,1],"capabl":1,"capacitydelta":1,"capitalon":1,"capon":1,"captur":0,"cardda_tagcompli":1,"cardin":1,"case":[0,1,5,6],"categori":[1,6],"caught":0,"causesresourc":1,"cbhfqv":1,"cbr":1,"cde":1,"cel":[1,5,6],"cel_activ":1,"cel_ast":1,"cel_env":1,"cel_ev":0,"cel_funct":6,"cel_pars":0,"cel_program":6,"cel_repl":0,"cel_sourc":6,"cel_test":6,"cel_trac":[0,3],"celbyt":0,"celevalerror":0,"celfilt":[4,6],"celfunct":0,"celpars":0,"celparseerror":0,"celpi":[0,1,2,3,6,7],"celstr":0,"celsyntaxerror":0,"celtyp":[1,4,6,7],"celunsupportederror":0,"central":[0,6],"cert":1,"certain":[1,6],"certif":1,"cet":0,"cf":1,"cfn":1,"chain":[0,7],"chainmap":[0,7],"chang":1,"char":6,"check":[0,6],"check_custom_origin":1,"checker":6,"child":0,"childnr":0,"children":0,"choci":0,"choic":6,"choos":0,"chunk":1,"ci":1,"cidr_siz":1,"cidrblock":1,"cipher":1,"circuit":[0,6],"cl":0,"claim":6,"class":[0,1,3,6],"classic":1,"classmethod":0,"claus":[1,6],"clean":1,"clear":[1,6],"cli":[0,3,4],"client":[0,1,7],"clock":1,"clone":0,"close":[0,1,6],"cloud":[1,4],"cloudform":1,"cloudfront":1,"cloudhsm":1,"cloudsearch":1,"cloudtrailen":1,"cloudwatch":1,"cluster":1,"cm":1,"cm6aws11":1,"cmdbenviron":1,"cmdloop":0,"cml":1,"co":6,"code":[0,1,6],"codebuild":1,"codebuildproject":1,"codecommit":1,"codepipelin":1,"coercion":[0,6],"collect":1,"collis":1,"color":1,"column":[0,1],"com":[0,1,6],"combin":[0,1,6],"come":[0,1,6],"command":[0,2,3,5,7],"comment":1,"common":[0,2,6],"commun":1,"commut":0,"compar":1,"comparablevers":1,"comparison":[0,1,6],"compat":0,"compil":[0,1,5,6,7],"compiledrunn":[0,4,7],"complement":1,"complet":6,"completekei":0,"complex":[0,1,4,6],"compli":1,"compliant":[1,6],"complic":6,"compon":[0,1],"compons":6,"compos":1,"composit":1,"comput":[1,6],"computeenviron":1,"computesgfilt":1,"computesubnetfilt":1,"conceal":1,"concept":0,"concern":[0,2,6],"concret":0,"condit":[0,1],"conditionaland":0,"conditionalor":0,"configen":1,"configrul":1,"configur":[1,4,5],"confus":6,"connect":1,"connectionsecuritygroupfilt":1,"connectionsubnetfilt":1,"consid":[0,1,7],"consider":1,"consist":0,"consol":3,"consolid":0,"constant":0,"constraint":7,"construct":[0,4,6],"constructor":0,"contain":[0,1,3,6,7],"containerdefinit":1,"content":0,"context":[0,2,4],"continu":6,"control":1,"convers":[0,6,7],"convert":[0,1,6],"copi":[0,1],"core":[0,1],"correct":1,"could":[0,6],"count":6,"countnumb":1,"cours":0,"cover":0,"cpp":0,"cpu":1,"cpuutil":1,"crash":6,"creat":[0,1,6,7],"created":1,"createdtim":1,"creation":[0,1],"creationd":1,"creationtimestamp":6,"credentialreport":1,"credentialreportmixin":1,"crossaccountaccess":1,"crossaccountaccessfilt":1,"crossaccountaccessmixin":1,"crossaccountfilt":1,"crossaccountp":1,"crossregiontransf":1,"crypto":1,"csv":[0,1],"csv2dict":1,"ct":1,"ctl":1,"current":[0,1,6,7],"custodian":[1,4],"custodian_asv":1,"custodian_cleanup":1,"custodian_downtim":1,"custodian_invalid":1,"custodian_invalid_asg":1,"custodian_rds_offhours_ct":1,"custodian_rds_offhours_et":1,"custodian_rds_offhours_pt":1,"custodian_res":1,"custodian_s3_ns_templ":1,"custodian_snapshot":1,"custodian_statu":1,"custodian_stop":1,"custodian_tag":1,"custom":1,"cw":1,"d":[0,1,2,3,6],"dai":1,"daili":1,"darwin":5,"data":[0,2,4,6],"databas":[0,1],"databaseconnect":1,"dataev":1,"datapipelin":1,"datapoint":1,"date":1,"datetim":[0,1,6,7],"dateutil":0,"dax":1,"daxsecuritygroupfilt":1,"daxsubnetfilt":1,"daylight":1,"dbclusteridentifi":1,"dbsubnetgroup":1,"dbsubnetgroupnam":1,"dd":1,"deal":[0,6],"debug":[0,3],"decid":6,"decim":0,"decis":1,"decl":[0,1,6],"declar":[0,1,6],"decompos":0,"decor":[0,3],"deep":0,"deepli":0,"def":[1,6],"default":[0,6,7],"default_tz":1,"default_vpc":1,"defaultvpc":1,"defin":[0,1,2,7],"definit":[0,2,6],"deleg":0,"delet":[1,7],"deleteprotect":6,"delimit":1,"deliv":1,"demand":1,"deni":1,"denial":1,"depend":[0,1,4,5,7],"deprec":[1,7],"descend":0,"describ":[0,1,6],"describe_db_snapshot_attribut":1,"describe_subscription_filt":1,"descript":[0,1],"design":4,"desir":[0,1],"desk":2,"destin":1,"detail":[1,3,6],"detect":1,"determin":[1,6],"dev":[1,6],"develop":0,"dhcpoptionsfilt":1,"dhe":1,"dict":[0,1,7],"dict_find_nam":0,"dictionari":[0,1],"did":0,"differ":[0,1,4,6],"difficult":1,"digest":1,"digit":0,"dimens":1,"directconnect":1,"directli":[0,1,2],"directori":[1,3],"directorysecuritygroupfilt":1,"directorysubnetfilt":1,"directoryvpcfilt":1,"disabl":[0,1],"disableapitermin":1,"discard":0,"discov":0,"dispatch":6,"displai":0,"displaynam":1,"distinct":0,"distinguish":0,"distribut":1,"diverg":1,"divid":0,"dividebyzeroerror":6,"divis":0,"dlm":1,"dm":1,"dn":1,"dnshostnam":1,"dnsname":1,"dnssupport":1,"do":[0,1,2,3,6],"do_by":0,"do_exit":0,"do_quit":0,"do_set":0,"do_show":0,"doc":[0,1],"document":[0,1,6],"doe":[0,1,2,6,7],"doesn":[0,1,6],"doin":0,"domain":[0,1],"don":[0,1,2,6],"done":[0,1],"dot_id":0,"dot_ident_arg":0,"doubl":[0,2],"double_valu":0,"doubletyp":[0,7],"doublevalu":0,"down":[0,1,6],"dpb":0,"dsl":1,"due":1,"dump":0,"dumpast":0,"duplic":0,"durat":[0,1,6],"durationtyp":[0,1,7],"dure":[0,1,5],"dyn":0,"dynamodb":1,"dynamodbacceler":1,"e":[0,1],"e1":0,"e2":0,"each":[0,1,6],"easier":1,"easili":4,"east":1,"east1":6,"eastern":1,"ebsoptim":1,"ec":1,"ec2":[1,6],"ec2_userdata_stop":1,"ecdh":1,"ecdsa":1,"echo":2,"ecr":1,"ecrcrossaccountaccessfilt":1,"ecsclust":1,"ecsmetr":1,"ef":1,"effect":[0,1],"egress_viol":1,"eip":1,"either":[0,1],"ek":1,"ekssgfilt":1,"ekssubnetfilt":1,"eksvpcfilt":1,"elast":1,"elasticach":1,"elasticachesnapshot":1,"elasticachesnapshotag":1,"elasticbeanstalk":1,"elasticfilesystemmounttarget":1,"elasticmapreduc":1,"elasticsearch":1,"elasticsearchdomain":1,"elb":1,"elblog":1,"elbsecuritypolici":1,"elem":0,"element":[1,6],"elif":0,"elimin":0,"els":0,"email":[1,6],"email_verifi":6,"emb":2,"embed":[0,4,6],"emit":7,"emmo":1,"emphat":6,"empti":[0,1,3,7],"emr":1,"emrclust":1,"emrmetr":1,"enabl":[0,3,4,6],"enasupport":1,"encount":0,"encryptionenabledfilt":1,"end":[0,1,7],"endpoint":1,"endpointcrossaccountfilt":1,"endpointsecuritygroupfilt":1,"endpointsubnetfilt":1,"endpointvpcfilt":1,"engin":[0,1],"eni":1,"ensur":[1,6],"ent":1,"enter":0,"enterpris":1,"entir":0,"enum":[0,1],"enumtyp":0,"env":[0,1,6],"environ":[0,1,3,4,5,6,7],"environment":0,"eof":2,"ephemeralinstancefilt":1,"eq":1,"eq_test":0,"equal":[0,1],"equival":0,"err":[0,6],"erron":0,"error":[0,4],"error_text":0,"escap":0,"eschew":1,"essenc":6,"essenti":[0,1,4],"est":1,"establish":0,"et":1,"etc":[0,1],"eu":1,"europ":0,"eustac":0,"eval":[0,6,7],"eval_error":0,"eval_filt":1,"evalerror":6,"evalu":[1,2,3,4,6,7],"evauat":0,"even":[0,1,6],"event":0,"eventrul":1,"eventrulemetr":1,"eventruletarget":1,"eventu":6,"ever":6,"everi":[0,1],"everydai":1,"everyone_onli":1,"everyth":0,"evluat":6,"ex":[0,6],"exact":1,"exactli":6,"examin":[1,6],"exampl":[2,3,4],"exc_class":0,"exceed":1,"except":[0,1,4,7],"exceptionmanualsnapshot":1,"exclud":1,"exclude_imag":1,"exclus":6,"exec":2,"execut":[0,1,6],"exempt":1,"exist":[0,1,3],"exit":0,"expand":0,"expans":0,"expcept":0,"expect":[0,1],"expir":1,"explicit":[0,1],"explicitli":[0,1],"export":3,"expos":[0,1,6],"expost":0,"expostur":6,"expr":[0,1,2,6],"express":[0,1,2,5,6,7],"exprlist":0,"exprpb":6,"extant":1,"extend":[0,1,6],"extended_name_path":0,"extens":[0,1,6,7],"extent":[0,1,6],"extern":[0,7],"extra":1,"extract":[0,1],"f":[0,1,6],"factfind":1,"factori":0,"fail":[0,1],"failur":[0,1],"fairli":0,"fall":0,"fallback":0,"fals":[0,1,2,6],"far":0,"fargat":1,"fatalf":[0,6],"fatalln":[0,6],"faulttolerantsnapshot":1,"favor":0,"featur":[0,1],"fetch":1,"few":[0,1,6],"ffffffff":1,"field":[0,1,6],"fieldinit":0,"file":[0,1,3],"filter":[0,4],"filter_inst":6,"filter_registri":1,"filter_resource_statist":1,"filterrestintegr":1,"filterrestmethod":1,"final":[0,1,6,7],"find":0,"find_nam":0,"findid":0,"firehos":1,"first":[0,6],"fit":1,"fix":[0,1,6],"fixedint":0,"fixedtz":0,"fklter":1,"flag":1,"flat":0,"flavor":0,"fleet":1,"float":[0,2,7],"floatvalu":0,"flow_log":1,"flowlogfilt":1,"fmt":[0,6],"fnmatch":1,"focus":1,"follow":[0,1,3,6,7],"foo":1,"foobar":1,"forbidden":1,"form":[0,1,6],"formal":[0,6],"format":[0,1,3,6],"formatt":3,"formerli":0,"forwardref":0,"found":[0,1],"four":[0,1],"fraction":0,"frame":0,"framework":[6,7],"fri":1,"friendli":0,"from":[0,1,2,3,4,7],"fromport":1,"front":0,"frozenset":0,"fs_analytical_dev":1,"fs_analytical_qa":1,"fs_core_cas_qa":1,"fs_custodian_tag":1,"fs_manual_ebs_snapshot_expir":1,"fs_manual_rds_snapshot_expir":1,"fsx":1,"fulfil":0,"full_control":1,"fulli":0,"func":[0,6],"function":[0,2,4,7],"function_ev":0,"function_match":0,"function_s":0,"functiontyp":[0,6],"further":0,"futur":0,"g":[0,1],"g4":0,"gamelift":1,"gatewai":1,"gather":[0,1,6],"gb":1,"gc":1,"gcm":1,"ge":1,"gen":0,"gener":[0,1,6],"geograph":1,"get":[0,1],"get_access_log":1,"get_account":1,"get_credential_report":1,"get_dbsubnet_group_us":1,"get_endpoint":1,"get_flow_log":1,"get_health_ev":1,"get_instance_imag":1,"get_key_polici":1,"get_launch_configuration_nam":1,"get_load_balanc":1,"get_matching_alias":1,"get_metr":1,"get_metric_statist":1,"get_model":1,"get_opt":0,"get_orgid":1,"get_protocol":1,"get_raw_health_ev":1,"get_raw_metr":1,"get_rel":1,"get_related_id":1,"get_resource_polici":1,"get_resource_statist":1,"get_resource_valu":1,"get_type_protect":1,"get_vpc":1,"getdat":[0,1],"getdayofmonth":0,"getdayofweek":[0,1],"getdayofyear":0,"getfullyear":0,"gethour":[0,1],"getmillisecond":0,"getminut":0,"getmonth":0,"getsecond":0,"gettz":0,"gib":1,"github":[0,1,6],"give":6,"given":[0,1,6,7],"gl":1,"glacier":1,"glaciercrossaccountaccessfilt":1,"glob":1,"global":0,"globalgrantsfilt":1,"glue":1,"glueconnect":1,"go":[0,1,4],"go_mod":0,"goal":0,"golang":0,"goo":1,"good":[1,6],"googl":[0,1,2,6],"gracefulli":0,"grai":1,"grammar":0,"grantcount":1,"great":6,"greater":1,"greet":6,"greetfunc":6,"group":6,"groupid":1,"groupmembership":1,"groupnam":1,"grouppolicysizequota":1,"groupset":1,"groupsperuserquota":1,"groupsquota":1,"gt":[0,1],"gte":1,"h":[0,1],"ha":[0,2,4,6,7],"had":1,"hand":6,"handl":[0,1,6,7],"handler":3,"hapg":1,"happen":0,"hash":0,"hashabl":0,"hasn":7,"hasstatementfilt":1,"hasvirtualmfa":1,"have":[0,1,3,4,6],"haven":1,"head":1,"header":[1,6],"healtch":1,"health_ev":1,"healthcheckprotocolmismatch":1,"healthev":1,"healtheventfilt":1,"healthfilt":1,"heavi":1,"hello":[0,6],"help":[0,1,2,6],"here":[0,1,6],"heredoc":2,"hexadecim":0,"hh":0,"hidden":0,"higher":1,"highli":1,"hint":[0,7],"hit":1,"holidai":1,"home":[1,3],"hook":0,"host":[0,1],"hostedzon":1,"hostnam":1,"hour":[0,1],"hourli":1,"housekeep_unused_sg":1,"how":[1,6],"howev":[0,1,6],"hsm":1,"html":[0,1],"http":[0,1,4,6],"i":[0,2,4,5,6,7],"iamaccessfilt":1,"iamgroupinlinepolici":1,"iamgroupus":1,"iaminstanceprofil":1,"iamroleinlinepolici":1,"iamroleusag":1,"iamroleusagemixin":1,"iamsummari":1,"iamuserinlinepolici":1,"iana":[0,1],"idea":0,"ideal":0,"ident":[0,1],"ident_arg":0,"ident_pat":0,"ident_valu":0,"identif":0,"identifi":[0,1],"ie":1,"ietf":0,"ifi":0,"ignor":[0,1,6],"imageag":1,"imageagefilt":1,"imagefilt":1,"imageid":1,"imagenam":1,"imagesunusedmixin":1,"imageunusedfilt":1,"immedi":1,"implement":[0,2,4,7],"implent":1,"impli":[0,1],"implicit":1,"implicitli":1,"import":[0,1,2,4,6],"in_tre":0,"includ":[0,1,6,7],"incomplet":0,"incorrect":1,"incorrectlambda":1,"increas":0,"inde":6,"index":[0,4],"indexerror":0,"indic":0,"individu":[0,1],"infer":1,"infin":0,"info":1,"inform":[1,3],"infrastructur":4,"init":0,"initi":[0,7],"inject":0,"input":[0,6],"insid":0,"instal":[0,4],"instanc":[0,6],"instance_profile_usag":1,"instanceagefilt":1,"instanceattribut":1,"instancecreatetim":1,"instanceimag":1,"instanceimagebas":1,"instanceimagemixin":1,"instanceinitiatedshutdownbehavior":1,"instanceoffhour":1,"instanceonhour":1,"instanceprofil":1,"instanceprofilenam":1,"instanceprofilesquota":1,"instances":6,"instancestorag":1,"instancetyp":1,"instead":[0,1],"insufficient_data":1,"int":[0,1,2,3,7],"int32":0,"int32valu":[0,4],"int64":0,"int64_overflow_neg":0,"int64_overflow_posit":0,"int64_valu":0,"int64valu":0,"integ":[0,1],"integer_math":0,"integr":[4,7],"intend":0,"intent":[1,2,4],"interest":4,"interfac":[0,1,6],"interfacesecuritygroupfilt":1,"interfacesubnetfilt":1,"interfacevpcfilt":1,"intermedi":6,"intern":[0,1,7],"internet":1,"interoper":4,"interpret":0,"interpretedrunn":[0,4,7],"intersect":1,"intiial":0,"intoper":0,"intro":0,"introduc":0,"introspect":6,"inttyp":[0,7],"invalidconfigfilt":1,"invers":1,"invis":1,"involv":0,"io":0,"iop":1,"iot":1,"ip":1,"ippermiss":1,"ippermissionegress":1,"is_log":[],"islog":1,"isloggingfilt":1,"isn":[0,1,3,4],"isnotloggingfilt":1,"isqueryloggingen":1,"isrm":1,"iss":[0,6],"isshadow":1,"isshieldprotect":1,"isshieldprotectedmixin":1,"issslfilt":1,"issu":[0,1,6],"iswafen":1,"item":[0,1,6],"iter":[0,1],"its":[0,1,6],"itself":0,"iunjpz":1,"jmes_path":1,"jmes_path_map":1,"jmespath":1,"job":[0,1],"joda":0,"jq":[0,2],"json":[0,1,2,6],"json_to_cel":[1,6],"jsonlin":0,"just":1,"kai":0,"keep":[0,1],"kei":[0,4,6,7],"kernel":1,"key_valu":1,"keyalia":1,"keyerror":0,"keyid":1,"keynam":1,"keyrotationen":1,"keyrotationstatu":1,"kind":[0,1,6,7],"kinesi":1,"kms_alia":1,"kms_kei":1,"kmsfilter":1,"kmskeyalia":1,"known":0,"kwarg":0,"laast":0,"lambda":[0,1],"lambdacrossaccountaccessfilt":1,"lambdaeventsourc":1,"lambdalayervers":1,"langdef":[0,1],"languag":[0,1],"larg":1,"large_instance_typ":1,"lark":[0,7],"last":0,"last_rot":1,"last_used_d":1,"last_used_region":1,"last_used_servic":1,"lastrot":1,"lastupdatedtim":1,"lastwritedai":1,"later":[0,1],"latestsnapshot":1,"launchconfig":1,"launchconfigag":1,"launchconfigfilt":1,"launchconfigurationnam":1,"launchtyp":1,"layer":[1,7],"layercrossaccount":1,"lc_userdata":1,"le":1,"lead":[0,1,6],"least":[0,1],"leav":1,"left":[0,1],"legaci":1,"len":[0,1],"length":0,"less":[0,1,6],"let":6,"level":[0,1,3],"levelnam":3,"leverag":0,"lexic":0,"lh":6,"librari":[0,1,6],"licens":1,"life":[0,1],"lifecyclerul":1,"like":[0,1,2,3,6],"likewis":1,"limit":0,"line":[0,1,3,7],"lineno":3,"linkag":1,"list":[0,1,6,7],"list_lit":0,"list_typ":0,"listtyp":[0,7],"listvalu":0,"liter":[0,1],"littl":0,"ll":[0,6],"load":[0,1],"load_annot":0,"load_valu":0,"local":[0,1,3,7],"localized_ev":0,"locat":0,"log":[0,3,6],"logcrossaccountfilt":1,"logg":1,"logger":[0,3],"loggroup":1,"logic":[0,2,6],"logical_and":0,"logical_condit":0,"logical_not":0,"logical_or":0,"logtarget":1,"long":[0,1],"longer":1,"longest":0,"longtz":0,"look":[0,1,6],"lookup":0,"loos":6,"los_angel":1,"loss":1,"lot":0,"lower":1,"lowercas":0,"lt":[0,1],"lte":1,"m":[0,1,2,5,6],"m1":6,"m3":1,"machin":1,"machineri":6,"macro":[0,1,7],"macro_has_ev":0,"made":[0,1],"magnet":1,"magnitud":0,"mai":[0,1,5,6,7],"maid":1,"maid_offhour":1,"maid_statu":1,"main":0,"major":1,"make":[0,1,2,6],"mani":[0,1],"manual":1,"map":[0,1,6],"map_lit":0,"map_typ":0,"mapinit":0,"mapkeytyp":7,"mappubliciponlaunch":1,"maptyp":[0,1,6,7],"march":0,"mark":0,"marked_kei":1,"markedforop":1,"mask":6,"master":[0,1,6],"match":[0,1,5,6],"matchabl":1,"math":[0,2],"matter":0,"max":1,"maximum":1,"maxproperti":1,"maxsecond":0,"mayb":0,"mccarthi":0,"md":[0,1,6],"me":6,"mean":[0,1],"meaning":0,"medium":1,"meet":6,"member":[0,1],"member_dot":0,"member_dot_arg":0,"member_index":0,"member_item":0,"member_object":0,"mere":0,"messag":[0,1,2,3,6],"message_liter":0,"messagebrok":1,"messagetyp":0,"method":[0,3,6],"method_ev":0,"method_id":0,"metricsfilt":1,"mfa_act":1,"mfadevic":1,"mfadevicesinus":1,"mi":1,"middl":0,"might":[0,6],"mimic":2,"min":1,"minim":[3,4],"minimum":[0,1],"minimumpasswordlength":1,"minproperti":1,"minsecond":0,"minut":[0,1],"mismatch":0,"mismatchs3origin":1,"miss":0,"missingrout":1,"mistak":1,"mix":0,"mixin":1,"ml":1,"mm":[0,1],"mock":1,"mod":[0,1],"mode":0,"model":[0,1,6],"modern":1,"modif":1,"modifi":1,"modifyablevolum":1,"modul":[0,1,3,4],"modulo":0,"modulu":0,"moment":1,"mon":1,"monitor":1,"monthli":1,"more":[0,1,2,4,6],"morn":1,"most":[0,1,2,7],"mount":1,"mq":1,"mqmetric":1,"mqsgfilter":1,"mqsubnetfilt":1,"mssage":1,"multi":1,"multipl":[0,1,6],"multipli":1,"multiplication_div":0,"multiplication_mod":0,"multiplication_mul":0,"must":[0,1,3,6],"my":1,"my_extens":6,"my_extns":6,"mydata":1,"n":[0,2,3,6],"nacl":1,"name":[0,1,3,6,7],"name1":0,"name2":0,"name_arg":6,"name_token":0,"namecontain":0,"namespac":1,"nano":0,"nanosecondspersecond":0,"narrowli":1,"nat":1,"nativ":[0,1],"navgiat":0,"navig":0,"nc":[0,1],"nc1":0,"nc2":0,"ndjson":0,"ne":1,"necessari":6,"necessarili":0,"need":[0,1,4,6,7],"needless":0,"neg":0,"nest":[0,6],"nested_activ":0,"net":[0,1],"networkacl":1,"networkin":1,"networkinterfac":1,"networkloc":1,"networkout":1,"new":[0,1],"new_activ":0,"new_text":0,"newenv":[0,6],"newer":6,"newfunct":6,"newinstanceoverload":6,"newli":1,"newoverload":6,"newvar":[0,6],"next":[0,1,6,7],"ni":1,"nice":[0,6],"nightli":1,"nil":[0,6],"no_such_field":0,"node":0,"nodesnaphot":1,"nomin":0,"non":[0,3,4,6],"non_compli":1,"none":[0,1,6],"nonetyp":0,"nonpubl":1,"noreturn":0,"normal":[0,1],"nospecificiamrolemanagedpolici":1,"not_applic":1,"not_nul":1,"notabl":2,"notact":1,"note":[0,1,6],"notebook":1,"notebookinst":1,"notebooksecuritygroupfilt":1,"notebooksubnetfilt":1,"notencryptedfilt":1,"notfound":0,"noth":[0,1,6],"notifi":1,"notimplementederror":6,"notprincip":1,"notresourc":1,"nov":1,"now":[0,1,6],"nsj7vg":1,"ntp":1,"null":[0,1],"null_typ":0,"null_valu":0,"nulltyp":[0,7],"nullvalu":0,"number":[0,1,6,7],"number_valu":0,"o":[0,1,3],"object":[0,2,4,6],"obscur":1,"obsolet":0,"obvers":1,"occur":[1,6],"octal":0,"odd":0,"off":[0,1],"offer":[0,1],"offhours_except":1,"offset":[0,1],"often":[0,7],"ok":1,"old":1,"older":1,"omit":1,"onc":[0,1],"one":[0,1,4,6],"oneof":[0,1],"onli":[0,1,3],"onlin":1,"onlyport":1,"oop":[0,6],"op":6,"op_name_map":0,"opa":1,"open":[1,6],"openpolicyag":1,"opensourc":4,"oper":[0,1],"operand":[0,1],"operator_in":0,"opswork":1,"opt":1,"optim":[1,6],"option":[0,2,3,5],"order":[0,1,6],"ordinari":0,"oreo":1,"org":[0,1],"organ":[1,4],"organiz":1,"origin":0,"other":[0,1,2,5,6,7],"otherwis":0,"out":[0,1,6],"outlin":1,"output":[0,6],"outsid":[0,1,6],"over":[0,1,2],"overal":0,"overdraftlimit":6,"overdraftprotect":6,"overflow":0,"overlap":0,"overload":[0,6,7],"overloadd":0,"overrid":[0,1],"overridden":0,"overwhelm":1,"own":[0,1,6],"owner":[1,6],"ownercontact":1,"ownereid":1,"p":0,"paackag":0,"pacif":1,"packag":[0,5,6,7],"packagetyp":0,"page":4,"pair":[0,1],"paragraph":3,"param":0,"paramet":[0,3,6],"parameterfilt":1,"paren_expr":0,"parent":[0,1],"parent_it":0,"pari":0,"pars":[0,1],"parse_cidr":1,"parse_d":1,"parse_text":1,"parser":[1,4],"part":[0,1,4,6,7],"partial":[0,6],"particular":[1,2],"particularli":[1,2],"pass":[1,6],"password_en":1,"password_last_chang":1,"password_last_us":1,"password_next_rot":1,"passwordpolicyconfigur":1,"past":1,"patch":1,"path":[0,1],"pattern":[0,1],"payload":0,"peer":1,"peeringconnect":1,"pendulum":[0,1],"per":1,"percent":1,"perfectli":1,"perform":[0,1],"perhap":0,"period":[0,1],"permiss":1,"permissionsboundaryusagecount":1,"permit":[0,1,6,7],"pg":1,"pgm":1,"phase":0,"phd":1,"pick":0,"piec":1,"pingstatu":1,"pip":5,"place":[0,1],"plan":[0,1],"platform":[1,4,5],"platformnam":1,"platformvers":1,"pleas":1,"plu":[0,1],"pluck":0,"pm":1,"poetri":5,"point":[0,1,2],"polici":[4,6],"policiesquota":1,"policyassign":1,"policynam":1,"policysizequota":1,"policyversionsinus":1,"policyversionsinusequota":1,"polit":0,"pool":1,"pop":0,"popul":0,"popular":1,"port":1,"possibl":[0,1],"possibli":[0,1,6],"post":1,"power":2,"pragmat":6,"prcess":1,"precis":0,"prefer":0,"prefix":[0,1],"prefixlen":1,"preloop":0,"prepar":1,"prepare_queri":1,"presenc":[0,1],"present":1,"preserv":1,"pretti":0,"prevent":[0,1],"previou":[0,1],"previous":1,"prg":[0,6],"prgm":[0,6],"primari":0,"primarili":0,"primit":1,"princip":1,"principl":4,"print":0,"println":[0,6],"prior":6,"probabl":1,"problem":[0,1,6],"process":[0,1,6],"process_json_doc":0,"process_resource_set":1,"process_value_typ":1,"prod":6,"prodlog":1,"produc":0,"product":0,"productcod":1,"profil":1,"program":[0,1,4,6,7],"programopt":6,"progress":0,"project":[0,1,4,6],"promot":1,"prompt":0,"propag":1,"proper":[1,6],"properli":[5,6],"properti":[0,1],"propog":0,"propos":1,"proto":[0,6],"proto3pb":0,"protobuf":[0,1,2],"protobuf_typ":0,"protobyf":0,"protocol":[0,4],"protubuf":0,"provid":[1,2,3,6,7],"provis":1,"proxi":1,"pst":1,"pt":1,"pubfac":1,"public":0,"pull":[1,6],"purchas":1,"pure":0,"purpos":1,"push":0,"put":1,"py":[1,4],"pypi":0,"python":[0,1,5,6,7],"pythonpath":2,"qa":1,"qualifi":0,"qualified_identifier_resolution_uncheck":0,"queue":1,"quickli":4,"quietli":0,"quit":0,"quot":0,"quota":1,"quux":0,"r":[0,6],"r53domain":1,"radisti":1,"raft":1,"rais":[0,2,6],"ramdisk":1,"rang":[0,7],"rank":1,"rare":[0,1],"raw":1,"rd":1,"rdscluster":1,"rdsclustersnapshot":1,"rdsoffhour":1,"rdsonhour":1,"rdssnapshot":1,"rdssnapshotag":1,"rdssnapshotonhour":1,"rdssubnetgroup":1,"rdt":1,"re":[0,1,6],"re2":5,"reach":1,"read":[0,1,2],"read_acp":1,"readm":0,"readonli":1,"readonlyrootfilesystem":1,"readreplicadbinstanceidentifi":1,"readreplicasourcedbinstanceidentifi":1,"real":0,"realli":[0,2],"reason":[1,4],"receiv":6,"recent":[0,1],"recogn":0,"recommend":1,"recov":1,"recurs":0,"redact":1,"redshift":1,"redshiftsnapshot":1,"redshiftsnapshotag":1,"redshiftsnapshotcrossaccount":1,"reduc":[0,1],"ref":[0,1,6],"ref_to":0,"refactirubg":1,"refactor":[0,1],"refer":[0,1],"referenc":1,"reflect":[0,6],"refractor":0,"refresh_period":1,"refus":0,"regex":[0,1],"region":1,"regist":[0,1],"registri":[0,1],"regular":[1,5],"rehydr":1,"reject":1,"rel":0,"relat":[0,1,6],"relatedresourcefilt":1,"relatedresourcemixin":1,"relation_eq":0,"relation_g":0,"relation_gt":0,"relation_in":0,"relation_l":0,"relation_lt":0,"relation_n":0,"relationshiup":1,"releas":1,"reli":[0,1,6],"remain":1,"remaind":0,"remot":0,"remov":[1,7],"renam":1,"reorder":1,"repeat":0,"repeatedli":1,"repl":0,"replac":[0,1,6],"replicationinst":1,"report":1,"report_delai":1,"report_gener":1,"report_max_ag":1,"repr":0,"repres":[0,1],"represent":1,"request":[0,1,6],"requie":6,"requir":[0,4,6],"require_ssl":1,"requiredencryptedputobject":1,"requireencryptedputobject":1,"requirelowercasecharact":1,"requirenumb":1,"requiresslaccessrdt":1,"requiresymbol":1,"requireuppercasecharact":1,"rerefer":0,"reservedconcurr":1,"resili":1,"resiz":1,"resize_config":1,"resolut":[0,6,7],"resolv":[0,6],"resolve_nam":0,"resolve_vari":0,"resourc":[0,1],"resource_count":1,"resource_list":6,"resource_schedul":1,"resource_typ":[1,6],"resourcekmskeyalia":1,"resourcekmskeyaliasmixin":1,"respect":0,"restapi":1,"restapicrossaccount":1,"restor":[0,1],"restresourc":1,"restrict":1,"result":[0,1,6,7],"result_funct":0,"resum":1,"resut":0,"return":[0,1,6],"reus":1,"revers":[0,1],"revis":1,"rework":1,"rewrit":6,"rewritten":0,"rfc":0,"rfc3339":0,"rh":6,"rhymjmbbe2":1,"rich":0,"right":0,"role":1,"rolecrossaccountaccess":1,"rolenam":1,"rolepolicysizequota":1,"rolesquota":1,"root":[0,1,3],"root_scop":0,"rootdevicenam":1,"route53":1,"routet":1,"rrset":1,"rsa":1,"rule":[0,6,7],"rulestatu":1,"run":[0,1,4,6],"runnabl":7,"runner":[0,4,7],"runner_class":[0,1],"runtim":0,"s3bucketnam":1,"s3crossaccountfilt":1,"s3metric":1,"s3publicblock":1,"sagemak":1,"sai":[0,1],"said":0,"same":[0,1],"samplecount":1,"save":[0,1],"sc":1,"scale":[0,1],"scan":1,"scan_group":1,"scatter":1,"scenario":0,"schedul":1,"schedulepars":1,"schema":[1,6],"scope":0,"se":1,"search":[0,1,4],"searchabl":0,"sechema":1,"second":[0,1],"secondari":1,"secondarili":0,"secret":[1,6],"secretsmanag":1,"section":1,"secur":4,"securetransport":1,"security_group":1,"securitygroup":1,"securitygroupdifffilt":1,"securitygroupfilt":1,"securitygrouplockedfilt":1,"see":[0,1,6],"seem":[0,1,7],"select":[0,1],"selector":1,"selector_valu":1,"self":[0,1,6],"selfrefer":1,"semant":[0,1,4,7],"semicolon":1,"send":1,"sens":3,"sensibl":1,"sentinel":1,"separ":[0,1],"seper":1,"sequenc":[0,1,6],"serial":0,"serv":0,"server":1,"servercertif":1,"servercertificatesquota":1,"service_role_usag":1,"servicelimit":1,"servicemetr":1,"servicetaskdefinitionfilt":1,"sessioncontext":1,"sessionissu":1,"set":[0,1,2,3],"set_activ":0,"set_annot":1,"setup":1,"sever":[0,1,6],"sg":1,"sg_unus":1,"sgdefaultvpc":1,"sgusag":1,"sgusagemixin":1,"sha256":1,"sha384":1,"shadow":0,"shake":6,"shake_hand":6,"shake_hands_string_str":6,"shakefunc":6,"share":1,"shell":0,"shield_protect":1,"shield_subscript":1,"shielden":1,"shieldenabledmixin":1,"shop":1,"short":[0,6],"shorthand":1,"should":[0,1,6,7],"show":0,"shown":0,"shrink":1,"si":1,"sid":1,"sign":[0,7],"signifi":1,"significantli":5,"signingcertificatesperuserquota":1,"silenc":[0,1,6],"silent":[0,6],"similar":[0,1],"similarli":[0,1,6],"simpl":0,"simple_test":0,"simpledb":1,"simpler":1,"simpli":0,"simplifi":[0,6],"sinc":1,"singl":[0,1,6],"single_bool":0,"single_byt":0,"single_doubl":0,"single_dur":0,"single_fixed32":0,"single_fixed64":0,"single_float":0,"single_int32":0,"single_int64":0,"single_sfixed32":0,"single_sfixed64":0,"single_sint32":0,"single_sint64":0,"single_str":0,"single_timestamp":0,"single_uint32":0,"single_uint32_wrapp":0,"single_uint64":0,"singleton":[0,4,6],"singletonfilt":1,"singular":0,"situat":1,"size":[0,1,6],"size_parse_cidr":1,"skew":1,"skew_hour":1,"skier":1,"skip":0,"slice":6,"slight":1,"slightli":[0,1],"slighyli":6,"slower":1,"slurp":0,"sm":1,"smaller":0,"smi":1,"sn":1,"snapshotag":1,"snapshotcreatetim":1,"snapshotcrossaccountaccess":1,"snapshotid":1,"snapshotskipamisnapshot":1,"snapshottyp":1,"snapshotunusedfilt":1,"snapshotunusedmixin":1,"snowbal":1,"snscrossaccount":1,"snscrossaccountmixin":1,"so":[0,6],"some":[0,1,3,6,7],"some_dict":0,"someparam":1,"someth":[0,1],"sometim":0,"somev":1,"somewhat":6,"sophist":0,"sort":6,"sourc":0,"sourcedestcheck":1,"sourceforg":0,"span":1,"sparingli":1,"spawn":1,"spec":[0,1],"special":[0,1,6],"specif":[0,4,6],"specifi":1,"specificiamrolemanagedpolici":1,"speed":5,"spell":1,"spin":1,"spite":6,"spread":0,"sprintf":6,"sq":1,"sql":1,"sqlserveraudit":1,"sqscrossaccount":1,"squar":1,"src":2,"sriovnetsupport":1,"ssd":1,"ssh":1,"sslnegot":1,"sslpolici":1,"sslpolicyfilt":1,"sslv2":1,"sslv3":1,"ssmstatu":1,"st":1,"stack":[0,1],"stage":1,"stand":[0,1],"standard":[0,1,6],"start":[0,1,6],"startswith":6,"starttim":1,"state":6,"statement_id":1,"statetransitionag":1,"static":0,"statist":1,"statu":0,"status":1,"stdin":[0,2],"stdout":0,"step":[0,1,6],"stick":0,"stop":1,"storag":1,"store":1,"str":[0,1,7],"stream":[0,1],"streamhandl":3,"strictli":0,"string":[0,1,3],"string_greet_str":6,"string_valu":0,"stringtyp":[0,6,7],"stringvalu":0,"strip":[0,1],"strong":1,"strongli":1,"struct":0,"structpb":0,"structur":[0,1,4,6],"studi":1,"stuff":6,"sub":[0,1,6],"sub_activ":0,"sub_ev":0,"sub_eval_parti":0,"sub_evalu":0,"subclass":[0,1,6],"sublanguag":1,"subnetfilt":1,"subnetid":1,"subnetrout":1,"subscript":1,"subsequ":[0,1],"subset":7,"subst":1,"succesfulli":1,"success":[0,1,6],"suffic":1,"suffix":[0,1],"sugar":0,"suggest":1,"suit":1,"suitabl":1,"sum":1,"summari":4,"sun":1,"super":1,"superclass":[0,1],"suppli":0,"support":[0,1],"sure":1,"survei":1,"suspend":1,"suspendedprocess":1,"swallow":0,"swap":1,"syntact":0,"syntax":[0,1,6],"system":[0,4],"t":[0,1,2,3,4,6,7],"tab":0,"tabl":1,"tag":6,"tag_polici":6,"tag_policy_filt":6,"tagstatu":1,"take":1,"take_act":6,"taken":[1,6],"target":0,"target_engin":1,"targetfunc":0,"targetgroup":1,"tasktaskdefinitionfilt":1,"tb":0,"team_nam":1,"tech":1,"techniqu":6,"temp":1,"templat":1,"temporari":0,"ten":7,"tend":0,"ters":1,"tertiari":0,"test":[0,1,2,6],"test_all_typ":0,"testabl":1,"testalltyp":0,"text":[0,1],"text2":0,"text_from":1,"textproto_to_gherkin":[],"than":[0,1,6],"the_filt":1,"thei":[0,1,6],"them":[0,1],"thi":[0,1,2,3,4,5,6,7],"thing":[0,1],"think":0,"thirti":1,"those":[0,1,6],"three":[0,1,4],"threshold":1,"through":[0,1,6],"tidyup":1,"time":[0,1,4,6],"timedelta":[0,1,7],"timestamp":[0,1,6],"timestamptyp":[0,1,6,7],"timezon":[0,1,6],"tini":2,"titl":1,"titlecas":0,"tl":1,"tlq9fr":1,"tls12":1,"tlsv1":1,"todo":1,"token":0,"toler":[0,6],"toml":3,"tool":[4,5],"top":[0,1,7],"topic":[1,4],"tot":2,"total":1,"toward":0,"tpb":0,"trace":[0,3],"traceback":[0,6],"track":[0,6],"tradit":0,"traffic":1,"trail":1,"transact":6,"transform":[0,1,6,7],"transient":[0,7],"transit":1,"translat":1,"transpar":0,"travers":1,"treat":0,"tree":[0,6],"tree_dump":0,"tree_for_express":0,"tree_for_vari":0,"tri":1,"trigger":1,"trim":1,"tripl":0,"trivial":[0,1,4],"troublingli":1,"true":[0,1,3,6],"truncat":0,"trust":1,"try":[0,6],"tupl":0,"turn":[0,1],"tweet":6,"two":[0,1,6,7],"txt":[0,1],"type":[1,2,3,4],"type_match":0,"type_nam":0,"type_name_map":0,"type_schema":1,"typeerror":0,"typetyp":0,"tz":[0,1,6],"tz_alias":[0,1],"tz_name":0,"tz_name_lookup":0,"tz_offset_pars":0,"tz_pars":0,"tzinfo":[0,1],"tzutc":1,"u":[0,1,6],"u0001f431":0,"u270c":0,"ubuntu":1,"uint":0,"uint32":0,"uint32valu":0,"uint64":0,"uint64_overflow_neg":0,"uint64_overflow_posit":0,"uint64_valu":0,"uint64valu":0,"uinttyp":[0,6,7],"uk":1,"unari":0,"unary_minus_no_overload":0,"unary_neg":0,"unary_not":0,"unchosen":0,"uncomput":6,"undecid":0,"under":1,"underli":6,"underutil":1,"unencrypt":1,"unifi":[0,1],"union":[0,1],"uniqu":[1,7],"unique_s":1,"unit":0,"unless":1,"unlik":0,"unmark":1,"unmodifi":1,"unoptim":1,"unsign":[0,7],"unsupport":0,"untag":1,"until":0,"unusediampolici":1,"unusediamrol":1,"unusedinstanceprofil":1,"unusedlaunchconfig":1,"unusedrdssubnetgroup":1,"unusedsecuritygroup":1,"unusu":1,"unwieldi":1,"up":[0,1,5],"upcom":1,"updat":[0,1],"upgradeavail":1,"upper":0,"uptimefilt":1,"url":1,"us":[0,3,4,5,7],"usag":1,"usediampolici":1,"usediamrol":1,"usedinstanceprofil":1,"usedsecuritygroup":1,"user":3,"user_creation_tim":1,"useraccesskei":1,"userag":1,"usercredentialreport":1,"userdata":1,"userdatafilt":1,"userguid":1,"userident":1,"usermfadevic":1,"usernam":1,"userpolici":1,"userpolicysizequota":1,"usersquota":1,"usual":[0,3],"utc":[0,6],"utcnow":1,"util":1,"uv":5,"v":0,"val":6,"valid":0,"valid_key_typ":0,"validconfigfilt":1,"valu":[0,3,6,7],"valueerror":0,"valuefilt":1,"valuekv":1,"values_from":1,"var":[0,6],"vari":1,"variabl":[0,1,2,3,6,7],"variant":[0,1,6],"variat":1,"varieti":0,"variou":[0,1,6],"vault":1,"veri":[0,1,3],"verifi":1,"version":[0,1,3,6],"versionsperpolicyquota":1,"via":[0,1,6],"viabl":0,"virtual":5,"visibl":[0,6],"visit":0,"visit_children":0,"visual":6,"vm":1,"vol":1,"volum":1,"volumeid":1,"vpcconfig":1,"vpcendpoint":1,"vpcfilter":1,"vpcid":1,"vpcidfilt":1,"vpclink":1,"vpcsecuritygroupfilt":1,"vpn":1,"w":1,"wa":[0,1,6],"wafen":1,"wai":[0,1,6],"wait":1,"walk":[0,7],"want":[0,1,2,6],"warn":[0,1,3],"watch":1,"we":[0,1,2,6],"web":1,"web_acl":1,"webacl":1,"week":1,"weekdai":1,"weekend":1,"weekli":1,"well":[0,1],"west":1,"what":[0,2,6],"when":[0,1,5,7],"where":[0,1,2,5],"whether":[0,1,6],"which":[0,1,6],"while":[0,1,3],"whitelist":1,"whitelist_condit":1,"whitelist_endpoint":1,"whitelist_endpoints_from":1,"whitelist_from":1,"whitelist_orgid":1,"whitelist_orgids_from":1,"whitelist_protocol":1,"whitelist_protocols_from":1,"whitelist_vpc":1,"whitelist_vpc_from":1,"whitelist_vpce_from":1,"who":1,"whole":[4,6],"whose":1,"why":0,"wide":1,"win":1,"window":[1,6],"wire":1,"with_traceback":0,"withdraw":6,"within":[0,1],"without":[0,1,6],"won":0,"work":[0,1,6],"workabl":1,"worker":1,"workflow":1,"world":[0,6],"would":[0,1,6],"wrap":[0,6,7],"wrapper":0,"wrapperspb":0,"write":6,"write_acp":1,"writen":1,"wrt":1,"www":[0,1],"x":[0,1,2,3],"x_sign":0,"xrayencrypt":1,"xyz":1,"y":[0,1,3],"ye":[1,2],"yeah":0,"yet":7,"yield":[0,1],"you":[1,2,6],"your":1,"yyyi":1,"z":[0,1],"z0":0,"za":[0,1],"zero":0,"zerodivisionerror":0,"zone":[0,1,6],"\u00b5":0},"titles":["CEL-Py API","C7N Functions Required","CLI Use of CEL-Python","Configuration","Pure Python Google Common Expression Language (CEL)","Installation","Application Integration","Data Structures"],"titleterms":{"The":6,"__main__":0,"access":1,"account":1,"adapt":0,"ag":1,"alia":1,"all":1,"allow":1,"ami":1,"anoth":0,"api":0,"applic":6,"argument":0,"attribut":1,"avail":1,"baselin":6,"bind":6,"block":1,"bool":1,"boolean":1,"bucket":1,"builtin":6,"bulk":6,"c7n":[1,6],"cach":1,"capac":1,"cel":[0,2,4,7],"celfilt":1,"celtyp":0,"check":1,"cidr":1,"cli":2,"cloud":6,"cloudtrail":1,"common":[1,4],"complianc":1,"concurr":1,"config":1,"configur":3,"construct":1,"content":[1,4],"context":1,"convers":1,"count":1,"credenti":1,"cross":1,"custodian":6,"custom":6,"data":[1,7],"db":1,"default":1,"defin":6,"definit":1,"delta":1,"design":1,"detail":0,"devic":1,"dhcp":1,"diff":1,"eb":1,"egress":1,"enabl":1,"encrypt":1,"ephemer":1,"error":6,"essenti":6,"evalu":0,"event":1,"exampl":[0,1,6],"except":6,"express":4,"extern":1,"fault":1,"filter":[1,6],"find":1,"flow":1,"from":6,"function":[1,6],"global":[1,6],"go":6,"googl":4,"grant":1,"group":1,"ha":1,"health":1,"healthcheck":1,"i":1,"iam":1,"id":1,"imag":1,"implement":[1,6],"indic":4,"ingress":1,"inlin":1,"instal":5,"instanc":1,"integr":[1,6],"invalid":1,"inventori":1,"kei":1,"km":1,"languag":4,"last":1,"latest":1,"launch":1,"lifecycl":1,"limit":1,"listen":1,"locat":1,"lock":1,"log":1,"manag":1,"mark":1,"method":1,"metric":1,"mfa":1,"mismatch":1,"miss":1,"modify":1,"namespac":0,"network":1,"non":1,"notif":1,"numer":0,"object":1,"offhour":1,"onhour":1,"op":1,"oper":6,"option":1,"origin":1,"param":1,"paramet":1,"parser":0,"password":1,"polici":1,"principl":1,"progag":1,"protect":1,"protocol":1,"provid":0,"public":1,"pure":4,"py":0,"python":[2,4],"queri":1,"readm":6,"requir":1,"reserv":1,"resourc":6,"rest":1,"rotat":1,"rout":1,"rule":1,"run":7,"s3":1,"secur":1,"servic":1,"shadow":1,"shield":1,"simpl":6,"singleton":1,"skip":1,"snapshot":1,"sourc":1,"specif":1,"ssl":1,"ssm":1,"stale":1,"state":1,"statement":1,"statu":1,"string":6,"structur":7,"subnet":1,"summari":1,"synopsi":0,"tabl":4,"tag":1,"target":1,"task":1,"termin":1,"textproto_to_gherkin":[],"time":7,"timzon":0,"todo":[0,6],"toler":1,"tool":[],"type":[0,6,7],"unus":1,"upgrad":1,"uptim":1,"us":[1,2,6],"user":1,"valid":1,"valu":1,"value_from":1,"value_typ":1,"virtual":1,"vpc":1,"waf":1,"write":1,"xrai":1}}) \ No newline at end of file +Search.setIndex({"alltitles":{"API":[[0,null]],"Additional Functions":[[0,"additional-functions"]],"Application Integration":[[7,null]],"Architecture and Design":[[8,null]],"Arguments, Types, and Namespaces":[[2,"arguments-types-and-namespaces"]],"Baseline C7N Example":[[7,"baseline-c7n-example"]],"Bulk Filter Example":[[7,"bulk-filter-example"]],"C7N Cache":[[1,"c7n-cache"]],"C7N Context Object":[[0,"c7n-context-object"],[1,"c7n-context-object"]],"C7N Filter and Resource Types":[[7,"c7n-filter-and-resource-types"]],"C7N Functions Required":[[1,null]],"CEL Types":[[8,"cel-types"]],"CELFilter Design":[[1,"celfilter-design"]],"CLI Use of CEL-Python":[[2,null]],"CONFIGURATION":[[2,"configuration"]],"Cloud Custodian (C7N) Integration":[[7,"cloud-custodian-c7n-integration"]],"Common C7N Constructs":[[1,"common-c7n-constructs"]],"Common/Boolean Filters":[[1,"common-boolean-filters"]],"Common/Non-Bool Filters":[[1,"common-non-bool-filters"]],"Compile-Time":[[8,"compile-time"]],"Components":[[8,"components"]],"Configuration":[[3,null]],"Container":[[8,"container"]],"Contents":[[1,"contents"]],"Context":[[8,"context"]],"Custom function in Go":[[7,"custom-function-in-go"]],"DESCRIPTION":[[2,"description"],[4,"description"]],"Define custom global function":[[7,"define-custom-global-function"]],"Design Principles":[[1,"design-principles"]],"Development Tools":[[4,null]],"Documentation Content:":[[5,null]],"ENVIRONMENT VARIABLES":[[2,"environment-variables"]],"EXAMPLES":[[2,"examples"],[4,"examples"]],"EXIT STATUS":[[2,"exit-status"]],"Evaluation-Time":[[8,"evaluation-time"]],"Exceptions and Errors":[[7,"exceptions-and-errors"]],"FILES":[[2,"files"],[4,"files"]],"Function Bindings":[[7,"function-bindings"]],"Indices and tables":[[5,"indices-and-tables"]],"Installation":[[6,null]],"Integration Essentials":[[7,"integration-essentials"]],"Integration Overview":[[5,"integration-overview"]],"More Examples from Go implementation":[[7,"more-examples-from-go-implementation"]],"Name Resolution":[[0,"name-resolution"]],"Numeric Details":[[0,"numeric-details"]],"Pure Python Google Common Expression Language (CEL)":[[5,null]],"SYNOPSIS":[[2,"synopsis"],[4,"synopsis"]],"Simple example using builtin types":[[7,"simple-example-using-builtin-types"]],"Singleton/Boolean Filters":[[1,"singleton-boolean-filters"]],"Singleton/Non-Bool Filters":[[1,"singleton-non-bool-filters"]],"Summary":[[1,"summary"]],"The API":[[0,"the-api"]],"The Member-Dot Production":[[8,"the-member-dot-production"]],"The Project Makefile":[[4,"the-project-makefile"]],"The docs/Makefile":[[4,"the-docs-makefile"]],"The features/Makefile":[[4,"the-features-makefile"]],"The pb2g Tool":[[4,"the-pb2g-tool"]],"The type: value Features":[[0,"the-type-value-features"]],"The type: value_from features":[[0,"the-type-value-from-features"]],"Timzone Details":[[0,"timzone-details"]],"Todo":[[0,"id2"],[0,"id3"],[0,"id4"],[0,"id5"],[0,"id6"],[0,"id7"],[0,"id8"],[0,"id9"],[0,"id10"],[0,"id11"],[0,"id12"],[0,"id13"],[0,"id14"],[0,"id15"],[0,"id16"],[0,"id17"],[0,"id18"],[0,"id19"],[0,"id20"],[0,"id21"],[0,"id22"],[0,"id23"],[0,"id24"],[0,"id25"],[0,"id26"],[0,"id27"],[0,"id28"],[0,"id29"],[0,"id30"],[0,"id31"],[0,"id32"],[0,"id33"],[0,"id34"],[0,"id35"],[0,"id36"],[0,"id37"],[0,"id38"],[0,"id39"],[0,"id40"],[0,"id41"]],"Transpiler Missing Names":[[8,"transpiler-missing-names"]],"Type Adapter":[[0,"type-adapter"]],"Type Provider":[[0,"type-provider"]],"Types":[[0,"types"]],"__main__":[[0,"module-celpy.__main__"]],"access-key":[[1,"access-key"]],"adapter":[[0,"module-celpy.adapter"]],"age":[[1,"age"]],"bucket-encryption (no examples)":[[1,"bucket-encryption-no-examples"]],"bucket-notification (no examples)":[[1,"bucket-notification-no-examples"]],"c7nlib":[[0,"module-celpy.c7nlib"]],"capacity-delta":[[1,"capacity-delta"]],"celparser":[[0,"module-celpy.celparser"]],"celpy":[[0,"module-celpy.__init__"]],"celtypes":[[0,"module-celpy.celtypes"]],"check-cloudtrail":[[1,"check-cloudtrail"]],"check-config":[[1,"check-config"]],"config-compliance (no examples)":[[1,"config-compliance-no-examples"]],"credential":[[1,"credential"]],"cross-account":[[1,"cross-account"]],"data-events (no examples)":[[1,"data-events-no-examples"]],"db-parameter (no examples)":[[1,"db-parameter-no-examples"]],"default-vpc (no examples)":[[1,"default-vpc-no-examples"]],"dhcp-options (no examples)":[[1,"dhcp-options-no-examples"]],"diff (no examples)":[[1,"diff-no-examples"]],"ebs":[[1,"ebs"]],"egress":[[1,"egress"]],"ephemeral (no examples)":[[1,"ephemeral-no-examples"]],"evaluation":[[0,"module-celpy.evaluation"]],"event":[[1,"event"]],"event-source (no examples)":[[1,"event-source-no-examples"]],"fault-tolerant (no examples)":[[1,"fault-tolerant-no-examples"]],"finding (no examples)":[[1,"finding-no-examples"]],"flow-logs":[[1,"flow-logs"]],"global-grants":[[1,"global-grants"]],"grant-count":[[1,"grant-count"]],"group (no examples)":[[1,"group-no-examples"]],"has-allow-all (no examples)":[[1,"has-allow-all-no-examples"]],"has-inline-policy (no examples)":[[1,"has-inline-policy-no-examples"]],"has-specific-managed-policy (no examples)":[[1,"has-specific-managed-policy-no-examples"]],"has-statement":[[1,"has-statement"]],"has-users (no examples)":[[1,"has-users-no-examples"]],"has-virtual-mfa (no examples)":[[1,"has-virtual-mfa-no-examples"]],"health-event":[[1,"health-event"]],"healthcheck-protocol-mismatch (no examples)":[[1,"healthcheck-protocol-mismatch-no-examples"]],"iam-summary (no examples)":[[1,"iam-summary-no-examples"]],"image":[[1,"image"]],"image-age":[[1,"image-age"]],"ingress":[[1,"ingress"]],"instance (no examples)":[[1,"instance-no-examples"]],"instance-age":[[1,"instance-age"]],"instance-attribute (no examples)":[[1,"instance-attribute-no-examples"]],"instance-uptime":[[1,"instance-uptime"]],"invalid":[[1,"invalid"]],"inventory (no examples)":[[1,"inventory-no-examples"]],"is-log-target":[[1,"is-log-target"]],"is-logging":[[1,"is-logging"]],"is-not-logging":[[1,"is-not-logging"]],"is-shadow (no examples)":[[1,"is-shadow-no-examples"]],"is-ssl (no examples)":[[1,"is-ssl-no-examples"]],"key-rotation-status":[[1,"key-rotation-status"]],"kms-alias":[[1,"kms-alias"]],"kms-key":[[1,"kms-key"]],"last-write":[[1,"last-write"]],"latest":[[1,"latest"]],"launch-config":[[1,"launch-config"]],"lifecycle-rule (no examples)":[[1,"lifecycle-rule-no-examples"]],"listener":[[1,"listener"]],"locked (no examples)":[[1,"locked-no-examples"]],"marked-for-op":[[1,"marked-for-op"]],"metrics":[[1,"metrics"]],"mfa-device":[[1,"mfa-device"]],"mismatch-s3-origin":[[1,"mismatch-s3-origin"]],"missing":[[1,"missing"]],"missing-policy-statement":[[1,"missing-policy-statement"]],"missing-route (no examples)":[[1,"missing-route-no-examples"]],"modifyable (no examples)":[[1,"modifyable-no-examples"]],"network-location":[[1,"network-location"]],"no-encryption-statement (no examples)":[[1,"no-encryption-statement-no-examples"]],"no-specific-managed-policy (no examples)":[[1,"no-specific-managed-policy-no-examples"]],"not-encrypted":[[1,"not-encrypted"]],"offhour":[[1,"offhour"]],"onhour":[[1,"onhour"]],"op Implementations":[[1,"op-implementations"]],"param (no examples)":[[1,"param-no-examples"]],"password-policy":[[1,"password-policy"]],"policy":[[1,"policy"]],"progagated-tags (no examples)":[[1,"progagated-tags-no-examples"]],"query-logging-enabled (no examples)":[[1,"query-logging-enabled-no-examples"]],"reserved-concurrency":[[1,"reserved-concurrency"]],"rest-integration (no examples)":[[1,"rest-integration-no-examples"]],"rest-method (no examples)":[[1,"rest-method-no-examples"]],"route (no examples)":[[1,"route-no-examples"]],"s3-cidr (no examples)":[[1,"s3-cidr-no-examples"]],"s3-public-block (no examples)":[[1,"s3-public-block-no-examples"]],"security-group":[[1,"security-group"]],"service-limit":[[1,"service-limit"]],"shield-enabled":[[1,"shield-enabled"]],"shield-metrics (no examples)":[[1,"shield-metrics-no-examples"]],"singleton (no examples)":[[1,"singleton-no-examples"]],"skip-ami-snapshots":[[1,"skip-ami-snapshots"]],"ssl-policy":[[1,"ssl-policy"]],"ssm (no examples)":[[1,"ssm-no-examples"]],"stale (no examples)":[[1,"stale-no-examples"]],"state-age":[[1,"state-age"]],"status (no examples)":[[1,"status-no-examples"]],"subnet":[[1,"subnet"]],"tag-count":[[1,"tag-count"]],"target-group (no examples)":[[1,"target-group-no-examples"]],"task-definition (no examples)":[[1,"task-definition-no-examples"]],"termination-protected (no examples)":[[1,"termination-protected-no-examples"]],"unused":[[1,"unused"]],"upgrade-available (no examples)":[[1,"upgrade-available-no-examples"]],"used":[[1,"used"]],"user-data (no examples)":[[1,"user-data-no-examples"]],"valid":[[1,"valid"]],"value":[[1,"value"]],"value_from External Data":[[1,"value-from-external-data"]],"value_type Conversions":[[1,"value-type-conversions"]],"vpc":[[1,"vpc"]],"vpc-attributes (no examples)":[[1,"vpc-attributes-no-examples"]],"vpc-id":[[1,"vpc-id"]],"waf-enabled":[[1,"waf-enabled"]],"xray-encrypt-key (no examples)":[[1,"xray-encrypt-key-no-examples"]]},"docnames":["api","c7n_functions","cli","configuration","development","index","installation","integration","structure"],"envversion":{"sphinx":65,"sphinx.domains.c":3,"sphinx.domains.changeset":1,"sphinx.domains.citation":1,"sphinx.domains.cpp":9,"sphinx.domains.index":1,"sphinx.domains.javascript":3,"sphinx.domains.math":2,"sphinx.domains.python":4,"sphinx.domains.rst":2,"sphinx.domains.std":2,"sphinx.ext.todo":2,"sphinx.ext.viewcode":1},"filenames":["api.rst","c7n_functions.rst","cli.rst","configuration.rst","development.rst","index.rst","installation.rst","integration.rst","structure.rst"],"indexentries":{"__abstractmethods__ (celpy.__init__.compiledrunner attribute)":[[0,"celpy.__init__.CompiledRunner.__abstractmethods__",false]],"__abstractmethods__ (celpy.__init__.interpretedrunner attribute)":[[0,"celpy.__init__.InterpretedRunner.__abstractmethods__",false]],"__abstractmethods__ (celpy.__init__.runner attribute)":[[0,"celpy.__init__.Runner.__abstractmethods__",false]],"__abstractmethods__ (celpy.c7nlib.c7n_interpreted_runner attribute)":[[0,"celpy.c7nlib.C7N_Interpreted_Runner.__abstractmethods__",false]],"__abstractmethods__ (celpy.evaluation.evaluator attribute)":[[0,"celpy.evaluation.Evaluator.__abstractmethods__",false]],"__add__() (celpy.celtypes.durationtype method)":[[0,"celpy.celtypes.DurationType.__add__",false]],"__add__() (celpy.celtypes.inttype method)":[[0,"celpy.celtypes.IntType.__add__",false]],"__add__() (celpy.celtypes.timestamptype method)":[[0,"celpy.celtypes.TimestampType.__add__",false]],"__add__() (celpy.celtypes.uinttype method)":[[0,"celpy.celtypes.UintType.__add__",false]],"__add__() (celpy.evaluation.celevalerror method)":[[0,"celpy.evaluation.CELEvalError.__add__",false]],"__call__() (celpy.celtypes.functiontype method)":[[0,"celpy.celtypes.FunctionType.__call__",false]],"__call__() (celpy.evaluation.celevalerror method)":[[0,"celpy.evaluation.CELEvalError.__call__",false]],"__contains__() (celpy.c7nlib.ipv4network method)":[[0,"celpy.c7nlib.IPv4Network.__contains__",false]],"__enter__() (celpy.c7nlib.c7ncontext method)":[[0,"celpy.c7nlib.C7NContext.__enter__",false]],"__eq__() (celpy.c7nlib.comparableversion method)":[[0,"celpy.c7nlib.ComparableVersion.__eq__",false]],"__eq__() (celpy.celtypes.doubletype method)":[[0,"celpy.celtypes.DoubleType.__eq__",false]],"__eq__() (celpy.celtypes.inttype method)":[[0,"celpy.celtypes.IntType.__eq__",false]],"__eq__() (celpy.celtypes.listtype method)":[[0,"celpy.celtypes.ListType.__eq__",false]],"__eq__() (celpy.celtypes.maptype method)":[[0,"celpy.celtypes.MapType.__eq__",false]],"__eq__() (celpy.celtypes.nulltype method)":[[0,"celpy.celtypes.NullType.__eq__",false]],"__eq__() (celpy.celtypes.stringtype method)":[[0,"celpy.celtypes.StringType.__eq__",false]],"__eq__() (celpy.celtypes.uinttype method)":[[0,"celpy.celtypes.UintType.__eq__",false]],"__eq__() (celpy.evaluation.celevalerror method)":[[0,"celpy.evaluation.CELEvalError.__eq__",false]],"__eq__() (celpy.evaluation.referent method)":[[0,"celpy.evaluation.Referent.__eq__",false]],"__exit__() (celpy.c7nlib.c7ncontext method)":[[0,"celpy.c7nlib.C7NContext.__exit__",false]],"__floordiv__() (celpy.celtypes.inttype method)":[[0,"celpy.celtypes.IntType.__floordiv__",false]],"__floordiv__() (celpy.celtypes.uinttype method)":[[0,"celpy.celtypes.UintType.__floordiv__",false]],"__floordiv__() (celpy.evaluation.celevalerror method)":[[0,"celpy.evaluation.CELEvalError.__floordiv__",false]],"__ge__() (celpy.celtypes.inttype method)":[[0,"celpy.celtypes.IntType.__ge__",false]],"__ge__() (celpy.celtypes.listtype method)":[[0,"celpy.celtypes.ListType.__ge__",false]],"__getattr__() (celpy.evaluation.activation method)":[[0,"celpy.evaluation.Activation.__getattr__",false]],"__getitem__() (celpy.celtypes.maptype method)":[[0,"celpy.celtypes.MapType.__getitem__",false]],"__gt__() (celpy.celtypes.inttype method)":[[0,"celpy.celtypes.IntType.__gt__",false]],"__gt__() (celpy.celtypes.listtype method)":[[0,"celpy.celtypes.ListType.__gt__",false]],"__hash__ (celpy.c7nlib.comparableversion attribute)":[[0,"celpy.c7nlib.ComparableVersion.__hash__",false]],"__hash__ (celpy.celtypes.listtype attribute)":[[0,"celpy.celtypes.ListType.__hash__",false]],"__hash__ (celpy.celtypes.maptype attribute)":[[0,"celpy.celtypes.MapType.__hash__",false]],"__hash__ (celpy.celtypes.nulltype attribute)":[[0,"celpy.celtypes.NullType.__hash__",false]],"__hash__ (celpy.evaluation.celevalerror attribute)":[[0,"celpy.evaluation.CELEvalError.__hash__",false]],"__hash__ (celpy.evaluation.referent attribute)":[[0,"celpy.evaluation.Referent.__hash__",false]],"__hash__() (celpy.celtypes.booltype method)":[[0,"celpy.celtypes.BoolType.__hash__",false]],"__hash__() (celpy.celtypes.doubletype method)":[[0,"celpy.celtypes.DoubleType.__hash__",false]],"__hash__() (celpy.celtypes.inttype method)":[[0,"celpy.celtypes.IntType.__hash__",false]],"__hash__() (celpy.celtypes.stringtype method)":[[0,"celpy.celtypes.StringType.__hash__",false]],"__hash__() (celpy.celtypes.uinttype method)":[[0,"celpy.celtypes.UintType.__hash__",false]],"__init__() (celpy.__init__.compiledrunner method)":[[0,"celpy.__init__.CompiledRunner.__init__",false]],"__init__() (celpy.__init__.environment method)":[[0,"celpy.__init__.Environment.__init__",false]],"__init__() (celpy.__init__.runner method)":[[0,"celpy.__init__.Runner.__init__",false]],"__init__() (celpy.c7nlib.c7ncontext method)":[[0,"celpy.c7nlib.C7NContext.__init__",false]],"__init__() (celpy.celparser.celparseerror method)":[[0,"celpy.celparser.CELParseError.__init__",false]],"__init__() (celpy.celparser.celparser method)":[[0,"celpy.celparser.CELParser.__init__",false]],"__init__() (celpy.celparser.dumpast method)":[[0,"celpy.celparser.DumpAST.__init__",false]],"__init__() (celpy.celtypes.maptype method)":[[0,"celpy.celtypes.MapType.__init__",false]],"__init__() (celpy.celtypes.messagetype method)":[[0,"celpy.celtypes.MessageType.__init__",false]],"__init__() (celpy.evaluation.activation method)":[[0,"celpy.evaluation.Activation.__init__",false]],"__init__() (celpy.evaluation.celevalerror method)":[[0,"celpy.evaluation.CELEvalError.__init__",false]],"__init__() (celpy.evaluation.celsyntaxerror method)":[[0,"celpy.evaluation.CELSyntaxError.__init__",false]],"__init__() (celpy.evaluation.celunsupportederror method)":[[0,"celpy.evaluation.CELUnsupportedError.__init__",false]],"__init__() (celpy.evaluation.evaluator method)":[[0,"celpy.evaluation.Evaluator.__init__",false]],"__init__() (celpy.evaluation.namecontainer method)":[[0,"celpy.evaluation.NameContainer.__init__",false]],"__init__() (celpy.evaluation.phase1transpiler method)":[[0,"celpy.evaluation.Phase1Transpiler.__init__",false]],"__init__() (celpy.evaluation.phase2transpiler method)":[[0,"celpy.evaluation.Phase2Transpiler.__init__",false]],"__init__() (celpy.evaluation.referent method)":[[0,"celpy.evaluation.Referent.__init__",false]],"__init__() (celpy.evaluation.transpiler method)":[[0,"celpy.evaluation.Transpiler.__init__",false]],"__init__() (celpy.evaluation.transpilertree method)":[[0,"celpy.evaluation.TranspilerTree.__init__",false]],"__le__() (celpy.celtypes.inttype method)":[[0,"celpy.celtypes.IntType.__le__",false]],"__le__() (celpy.celtypes.listtype method)":[[0,"celpy.celtypes.ListType.__le__",false]],"__lt__() (celpy.celtypes.inttype method)":[[0,"celpy.celtypes.IntType.__lt__",false]],"__lt__() (celpy.celtypes.listtype method)":[[0,"celpy.celtypes.ListType.__lt__",false]],"__mod__() (celpy.celtypes.doubletype method)":[[0,"celpy.celtypes.DoubleType.__mod__",false]],"__mod__() (celpy.celtypes.inttype method)":[[0,"celpy.celtypes.IntType.__mod__",false]],"__mod__() (celpy.celtypes.uinttype method)":[[0,"celpy.celtypes.UintType.__mod__",false]],"__mod__() (celpy.evaluation.celevalerror method)":[[0,"celpy.evaluation.CELEvalError.__mod__",false]],"__mul__() (celpy.celtypes.inttype method)":[[0,"celpy.celtypes.IntType.__mul__",false]],"__mul__() (celpy.celtypes.uinttype method)":[[0,"celpy.celtypes.UintType.__mul__",false]],"__mul__() (celpy.evaluation.celevalerror method)":[[0,"celpy.evaluation.CELEvalError.__mul__",false]],"__ne__() (celpy.celtypes.doubletype method)":[[0,"celpy.celtypes.DoubleType.__ne__",false]],"__ne__() (celpy.celtypes.inttype method)":[[0,"celpy.celtypes.IntType.__ne__",false]],"__ne__() (celpy.celtypes.listtype method)":[[0,"celpy.celtypes.ListType.__ne__",false]],"__ne__() (celpy.celtypes.maptype method)":[[0,"celpy.celtypes.MapType.__ne__",false]],"__ne__() (celpy.celtypes.nulltype method)":[[0,"celpy.celtypes.NullType.__ne__",false]],"__ne__() (celpy.celtypes.stringtype method)":[[0,"celpy.celtypes.StringType.__ne__",false]],"__ne__() (celpy.celtypes.uinttype method)":[[0,"celpy.celtypes.UintType.__ne__",false]],"__neg__() (celpy.celtypes.booltype method)":[[0,"celpy.celtypes.BoolType.__neg__",false]],"__neg__() (celpy.celtypes.doubletype method)":[[0,"celpy.celtypes.DoubleType.__neg__",false]],"__neg__() (celpy.celtypes.inttype method)":[[0,"celpy.celtypes.IntType.__neg__",false]],"__neg__() (celpy.celtypes.uinttype method)":[[0,"celpy.celtypes.UintType.__neg__",false]],"__neg__() (celpy.evaluation.celevalerror method)":[[0,"celpy.evaluation.CELEvalError.__neg__",false]],"__new__() (celpy.__init__.int32value static method)":[[0,"celpy.__init__.Int32Value.__new__",false]],"__new__() (celpy.celtypes.booltype static method)":[[0,"celpy.celtypes.BoolType.__new__",false]],"__new__() (celpy.celtypes.bytestype static method)":[[0,"celpy.celtypes.BytesType.__new__",false]],"__new__() (celpy.celtypes.doubletype static method)":[[0,"celpy.celtypes.DoubleType.__new__",false]],"__new__() (celpy.celtypes.durationtype static method)":[[0,"celpy.celtypes.DurationType.__new__",false]],"__new__() (celpy.celtypes.inttype static method)":[[0,"celpy.celtypes.IntType.__new__",false]],"__new__() (celpy.celtypes.stringtype static method)":[[0,"celpy.celtypes.StringType.__new__",false]],"__new__() (celpy.celtypes.timestamptype static method)":[[0,"celpy.celtypes.TimestampType.__new__",false]],"__new__() (celpy.celtypes.typetype static method)":[[0,"celpy.celtypes.TypeType.__new__",false]],"__new__() (celpy.celtypes.uinttype static method)":[[0,"celpy.celtypes.UintType.__new__",false]],"__orig_bases__ (celpy.celtypes.listtype attribute)":[[0,"celpy.celtypes.ListType.__orig_bases__",false]],"__orig_bases__ (celpy.celtypes.maptype attribute)":[[0,"celpy.celtypes.MapType.__orig_bases__",false]],"__orig_bases__ (celpy.evaluation.namecontainer attribute)":[[0,"celpy.evaluation.NameContainer.__orig_bases__",false]],"__parameters__ (celpy.celparser.dumpast attribute)":[[0,"celpy.celparser.DumpAST.__parameters__",false]],"__parameters__ (celpy.celtypes.listtype attribute)":[[0,"celpy.celtypes.ListType.__parameters__",false]],"__parameters__ (celpy.celtypes.maptype attribute)":[[0,"celpy.celtypes.MapType.__parameters__",false]],"__parameters__ (celpy.celtypes.messagetype attribute)":[[0,"celpy.celtypes.MessageType.__parameters__",false]],"__parameters__ (celpy.celtypes.packagetype attribute)":[[0,"celpy.celtypes.PackageType.__parameters__",false]],"__parameters__ (celpy.evaluation.evaluator attribute)":[[0,"celpy.evaluation.Evaluator.__parameters__",false]],"__parameters__ (celpy.evaluation.namecontainer attribute)":[[0,"celpy.evaluation.NameContainer.__parameters__",false]],"__parameters__ (celpy.evaluation.phase1transpiler attribute)":[[0,"celpy.evaluation.Phase1Transpiler.__parameters__",false]],"__parameters__ (celpy.evaluation.phase2transpiler attribute)":[[0,"celpy.evaluation.Phase2Transpiler.__parameters__",false]],"__parameters__ (celpy.evaluation.transpilertree attribute)":[[0,"celpy.evaluation.TranspilerTree.__parameters__",false]],"__pow__() (celpy.evaluation.celevalerror method)":[[0,"celpy.evaluation.CELEvalError.__pow__",false]],"__radd__() (celpy.celtypes.durationtype method)":[[0,"celpy.celtypes.DurationType.__radd__",false]],"__radd__() (celpy.celtypes.inttype method)":[[0,"celpy.celtypes.IntType.__radd__",false]],"__radd__() (celpy.celtypes.timestamptype method)":[[0,"celpy.celtypes.TimestampType.__radd__",false]],"__radd__() (celpy.celtypes.uinttype method)":[[0,"celpy.celtypes.UintType.__radd__",false]],"__radd__() (celpy.evaluation.celevalerror method)":[[0,"celpy.evaluation.CELEvalError.__radd__",false]],"__repr__() (celpy.__init__.environment method)":[[0,"celpy.__init__.Environment.__repr__",false]],"__repr__() (celpy.__init__.runner method)":[[0,"celpy.__init__.Runner.__repr__",false]],"__repr__() (celpy.c7nlib.c7ncontext method)":[[0,"celpy.c7nlib.C7NContext.__repr__",false]],"__repr__() (celpy.celtypes.booltype method)":[[0,"celpy.celtypes.BoolType.__repr__",false]],"__repr__() (celpy.celtypes.bytestype method)":[[0,"celpy.celtypes.BytesType.__repr__",false]],"__repr__() (celpy.celtypes.doubletype method)":[[0,"celpy.celtypes.DoubleType.__repr__",false]],"__repr__() (celpy.celtypes.durationtype method)":[[0,"celpy.celtypes.DurationType.__repr__",false]],"__repr__() (celpy.celtypes.inttype method)":[[0,"celpy.celtypes.IntType.__repr__",false]],"__repr__() (celpy.celtypes.listtype method)":[[0,"celpy.celtypes.ListType.__repr__",false]],"__repr__() (celpy.celtypes.maptype method)":[[0,"celpy.celtypes.MapType.__repr__",false]],"__repr__() (celpy.celtypes.stringtype method)":[[0,"celpy.celtypes.StringType.__repr__",false]],"__repr__() (celpy.celtypes.timestamptype method)":[[0,"celpy.celtypes.TimestampType.__repr__",false]],"__repr__() (celpy.celtypes.uinttype method)":[[0,"celpy.celtypes.UintType.__repr__",false]],"__repr__() (celpy.evaluation.activation method)":[[0,"celpy.evaluation.Activation.__repr__",false]],"__repr__() (celpy.evaluation.celevalerror method)":[[0,"celpy.evaluation.CELEvalError.__repr__",false]],"__repr__() (celpy.evaluation.namecontainer method)":[[0,"celpy.evaluation.NameContainer.__repr__",false]],"__repr__() (celpy.evaluation.referent method)":[[0,"celpy.evaluation.Referent.__repr__",false]],"__rfloordiv__() (celpy.celtypes.inttype method)":[[0,"celpy.celtypes.IntType.__rfloordiv__",false]],"__rfloordiv__() (celpy.celtypes.uinttype method)":[[0,"celpy.celtypes.UintType.__rfloordiv__",false]],"__rfloordiv__() (celpy.evaluation.celevalerror method)":[[0,"celpy.evaluation.CELEvalError.__rfloordiv__",false]],"__rmod__() (celpy.celtypes.doubletype method)":[[0,"celpy.celtypes.DoubleType.__rmod__",false]],"__rmod__() (celpy.celtypes.inttype method)":[[0,"celpy.celtypes.IntType.__rmod__",false]],"__rmod__() (celpy.celtypes.uinttype method)":[[0,"celpy.celtypes.UintType.__rmod__",false]],"__rmod__() (celpy.evaluation.celevalerror method)":[[0,"celpy.evaluation.CELEvalError.__rmod__",false]],"__rmul__() (celpy.celtypes.inttype method)":[[0,"celpy.celtypes.IntType.__rmul__",false]],"__rmul__() (celpy.celtypes.uinttype method)":[[0,"celpy.celtypes.UintType.__rmul__",false]],"__rmul__() (celpy.evaluation.celevalerror method)":[[0,"celpy.evaluation.CELEvalError.__rmul__",false]],"__rpow__() (celpy.evaluation.celevalerror method)":[[0,"celpy.evaluation.CELEvalError.__rpow__",false]],"__rsub__() (celpy.celtypes.inttype method)":[[0,"celpy.celtypes.IntType.__rsub__",false]],"__rsub__() (celpy.celtypes.uinttype method)":[[0,"celpy.celtypes.UintType.__rsub__",false]],"__rsub__() (celpy.evaluation.celevalerror method)":[[0,"celpy.evaluation.CELEvalError.__rsub__",false]],"__rtruediv__() (celpy.celtypes.doubletype method)":[[0,"celpy.celtypes.DoubleType.__rtruediv__",false]],"__rtruediv__() (celpy.celtypes.inttype method)":[[0,"celpy.celtypes.IntType.__rtruediv__",false]],"__rtruediv__() (celpy.celtypes.uinttype method)":[[0,"celpy.celtypes.UintType.__rtruediv__",false]],"__rtruediv__() (celpy.evaluation.celevalerror method)":[[0,"celpy.evaluation.CELEvalError.__rtruediv__",false]],"__str__() (celpy.celtypes.booltype method)":[[0,"celpy.celtypes.BoolType.__str__",false]],"__str__() (celpy.celtypes.doubletype method)":[[0,"celpy.celtypes.DoubleType.__str__",false]],"__str__() (celpy.celtypes.durationtype method)":[[0,"celpy.celtypes.DurationType.__str__",false]],"__str__() (celpy.celtypes.inttype method)":[[0,"celpy.celtypes.IntType.__str__",false]],"__str__() (celpy.celtypes.timestamptype method)":[[0,"celpy.celtypes.TimestampType.__str__",false]],"__str__() (celpy.celtypes.uinttype method)":[[0,"celpy.celtypes.UintType.__str__",false]],"__sub__() (celpy.celtypes.inttype method)":[[0,"celpy.celtypes.IntType.__sub__",false]],"__sub__() (celpy.celtypes.timestamptype method)":[[0,"celpy.celtypes.TimestampType.__sub__",false]],"__sub__() (celpy.celtypes.uinttype method)":[[0,"celpy.celtypes.UintType.__sub__",false]],"__sub__() (celpy.evaluation.celevalerror method)":[[0,"celpy.evaluation.CELEvalError.__sub__",false]],"__truediv__() (celpy.celtypes.doubletype method)":[[0,"celpy.celtypes.DoubleType.__truediv__",false]],"__truediv__() (celpy.celtypes.inttype method)":[[0,"celpy.celtypes.IntType.__truediv__",false]],"__truediv__() (celpy.celtypes.uinttype method)":[[0,"celpy.celtypes.UintType.__truediv__",false]],"__truediv__() (celpy.evaluation.celevalerror method)":[[0,"celpy.evaluation.CELEvalError.__truediv__",false]],"absent() (in module celpy.c7nlib)":[[0,"celpy.c7nlib.absent",false]],"activation (class in celpy.evaluation)":[[0,"celpy.evaluation.Activation",false]],"addition() (celpy.celparser.dumpast method)":[[0,"celpy.celparser.DumpAST.addition",false]],"addition() (celpy.evaluation.evaluator method)":[[0,"celpy.evaluation.Evaluator.addition",false]],"addition() (celpy.evaluation.phase1transpiler method)":[[0,"celpy.evaluation.Phase1Transpiler.addition",false]],"addition_add() (celpy.celparser.dumpast method)":[[0,"celpy.celparser.DumpAST.addition_add",false]],"addition_sub() (celpy.celparser.dumpast method)":[[0,"celpy.celparser.DumpAST.addition_sub",false]],"all_dbsubenet_groups() (in module celpy.c7nlib)":[[0,"celpy.c7nlib.all_dbsubenet_groups",false]],"all_images() (in module celpy.c7nlib)":[[0,"celpy.c7nlib.all_images",false]],"all_instance_profiles() (in module celpy.c7nlib)":[[0,"celpy.c7nlib.all_instance_profiles",false]],"all_launch_configuration_names() (in module celpy.c7nlib)":[[0,"celpy.c7nlib.all_launch_configuration_names",false]],"all_scan_groups() (in module celpy.c7nlib)":[[0,"celpy.c7nlib.all_scan_groups",false]],"all_service_roles() (in module celpy.c7nlib)":[[0,"celpy.c7nlib.all_service_roles",false]],"all_snapshots() (in module celpy.c7nlib)":[[0,"celpy.c7nlib.all_snapshots",false]],"ambiguous_literals() (celpy.celparser.celparser static method)":[[0,"celpy.celparser.CELParser.ambiguous_literals",false]],"arg_type_value() (in module celpy.__main__)":[[0,"celpy.__main__.arg_type_value",false]],"arn_split() (in module celpy.c7nlib)":[[0,"celpy.c7nlib.arn_split",false]],"bool_eq() (in module celpy.evaluation)":[[0,"celpy.evaluation.bool_eq",false]],"bool_ge() (in module celpy.evaluation)":[[0,"celpy.evaluation.bool_ge",false]],"bool_gt() (in module celpy.evaluation)":[[0,"celpy.evaluation.bool_gt",false]],"bool_le() (in module celpy.evaluation)":[[0,"celpy.evaluation.bool_le",false]],"bool_lt() (in module celpy.evaluation)":[[0,"celpy.evaluation.bool_lt",false]],"bool_ne() (in module celpy.evaluation)":[[0,"celpy.evaluation.bool_ne",false]],"boolean() (in module celpy.evaluation)":[[0,"celpy.evaluation.boolean",false]],"booltype (class in celpy.celtypes)":[[0,"celpy.celtypes.BoolType",false]],"build_macro_eval() (celpy.evaluation.evaluator method)":[[0,"celpy.evaluation.Evaluator.build_macro_eval",false]],"build_reduce_macro_eval() (celpy.evaluation.evaluator method)":[[0,"celpy.evaluation.Evaluator.build_reduce_macro_eval",false]],"build_ss_macro_eval() (celpy.evaluation.evaluator method)":[[0,"celpy.evaluation.Evaluator.build_ss_macro_eval",false]],"bytestype (class in celpy.celtypes)":[[0,"celpy.celtypes.BytesType",false]],"c7n_interpreted_runner (class in celpy.c7nlib)":[[0,"celpy.c7nlib.C7N_Interpreted_Runner",false]],"c7ncontext (class in celpy.c7nlib)":[[0,"celpy.c7nlib.C7NContext",false]],"cel_eval() (celpy.__main__.cel_repl method)":[[0,"celpy.__main__.CEL_REPL.cel_eval",false]],"cel_parser (celpy.celparser.celparser attribute)":[[0,"celpy.celparser.CELParser.CEL_PARSER",false]],"cel_repl (class in celpy.__main__)":[[0,"celpy.__main__.CEL_REPL",false]],"cel_trace":[[0,"index-0",false]],"celbytes() (in module celpy.evaluation)":[[0,"celpy.evaluation.celbytes",false]],"celevalerror":[[0,"celpy.evaluation.CELEvalError",false]],"celjsondecoder (class in celpy.adapter)":[[0,"celpy.adapter.CELJSONDecoder",false]],"celjsonencoder (class in celpy.adapter)":[[0,"celpy.adapter.CELJSONEncoder",false]],"celparseerror":[[0,"celpy.celparser.CELParseError",false]],"celparser (class in celpy.celparser)":[[0,"celpy.celparser.CELParser",false]],"celpy.__init__":[[0,"module-celpy.__init__",false]],"celpy.__main__":[[0,"module-celpy.__main__",false]],"celpy.adapter":[[0,"module-celpy.adapter",false]],"celpy.c7nlib":[[0,"module-celpy.c7nlib",false]],"celpy.celparser":[[0,"module-celpy.celparser",false]],"celpy.celtypes":[[0,"module-celpy.celtypes",false]],"celpy.evaluation":[[0,"module-celpy.evaluation",false]],"celstr() (in module celpy.evaluation)":[[0,"celpy.evaluation.celstr",false]],"celsyntaxerror":[[0,"celpy.evaluation.CELSyntaxError",false]],"celunsupportederror":[[0,"celpy.evaluation.CELUnsupportedError",false]],"children (celpy.evaluation.transpilertree attribute)":[[0,"celpy.evaluation.TranspilerTree.children",false]],"clone() (celpy.evaluation.activation method)":[[0,"celpy.evaluation.Activation.clone",false]],"clone() (celpy.evaluation.namecontainer method)":[[0,"celpy.evaluation.NameContainer.clone",false]],"clone() (celpy.evaluation.referent method)":[[0,"celpy.evaluation.Referent.clone",false]],"comparableversion (class in celpy.c7nlib)":[[0,"celpy.c7nlib.ComparableVersion",false]],"compile() (celpy.__init__.environment method)":[[0,"celpy.__init__.Environment.compile",false]],"compiledrunner (class in celpy.__init__)":[[0,"celpy.__init__.CompiledRunner",false]],"conditionaland() (celpy.celparser.dumpast method)":[[0,"celpy.celparser.DumpAST.conditionaland",false]],"conditionaland() (celpy.evaluation.evaluator method)":[[0,"celpy.evaluation.Evaluator.conditionaland",false]],"conditionaland() (celpy.evaluation.phase1transpiler method)":[[0,"celpy.evaluation.Phase1Transpiler.conditionaland",false]],"conditionaland() (celpy.evaluation.phase2transpiler method)":[[0,"celpy.evaluation.Phase2Transpiler.conditionaland",false]],"conditionalor() (celpy.celparser.dumpast method)":[[0,"celpy.celparser.DumpAST.conditionalor",false]],"conditionalor() (celpy.evaluation.evaluator method)":[[0,"celpy.evaluation.Evaluator.conditionalor",false]],"conditionalor() (celpy.evaluation.phase1transpiler method)":[[0,"celpy.evaluation.Phase1Transpiler.conditionalor",false]],"conditionalor() (celpy.evaluation.phase2transpiler method)":[[0,"celpy.evaluation.Phase2Transpiler.conditionalor",false]],"contains() (celpy.c7nlib.ipv4network method)":[[0,"celpy.c7nlib.IPv4Network.contains",false]],"contains() (celpy.celtypes.listtype method)":[[0,"celpy.celtypes.ListType.contains",false]],"contains() (celpy.celtypes.maptype method)":[[0,"celpy.celtypes.MapType.contains",false]],"contains() (celpy.celtypes.stringtype method)":[[0,"celpy.celtypes.StringType.contains",false]],"credentials() (in module celpy.c7nlib)":[[0,"celpy.c7nlib.credentials",false]],"data (celpy.evaluation.transpilertree attribute)":[[0,"celpy.evaluation.TranspilerTree.data",false]],"decode() (celpy.adapter.celjsondecoder method)":[[0,"celpy.adapter.CELJSONDecoder.decode",false]],"default() (celpy.__main__.cel_repl method)":[[0,"celpy.__main__.CEL_REPL.default",false]],"default() (celpy.adapter.celjsonencoder method)":[[0,"celpy.adapter.CELJSONEncoder.default",false]],"describe_db_snapshot_attributes() (in module celpy.c7nlib)":[[0,"celpy.c7nlib.describe_db_snapshot_attributes",false]],"describe_subscription_filters() (in module celpy.c7nlib)":[[0,"celpy.c7nlib.describe_subscription_filters",false]],"dict_find_name() (celpy.evaluation.namecontainer static method)":[[0,"celpy.evaluation.NameContainer.dict_find_name",false]],"difference() (in module celpy.c7nlib)":[[0,"celpy.c7nlib.difference",false]],"display() (celpy.celparser.dumpast class method)":[[0,"celpy.celparser.DumpAST.display",false]],"do_bye() (celpy.__main__.cel_repl method)":[[0,"celpy.__main__.CEL_REPL.do_bye",false]],"do_eof() (celpy.__main__.cel_repl method)":[[0,"celpy.__main__.CEL_REPL.do_EOF",false]],"do_exit() (celpy.__main__.cel_repl method)":[[0,"celpy.__main__.CEL_REPL.do_exit",false]],"do_quit() (celpy.__main__.cel_repl method)":[[0,"celpy.__main__.CEL_REPL.do_quit",false]],"do_set() (celpy.__main__.cel_repl method)":[[0,"celpy.__main__.CEL_REPL.do_set",false]],"do_show() (celpy.__main__.cel_repl method)":[[0,"celpy.__main__.CEL_REPL.do_show",false]],"dot_ident() (celpy.celparser.dumpast method)":[[0,"celpy.celparser.DumpAST.dot_ident",false]],"dot_ident() (celpy.evaluation.phase1transpiler method)":[[0,"celpy.evaluation.Phase1Transpiler.dot_ident",false]],"dot_ident_arg() (celpy.celparser.dumpast method)":[[0,"celpy.celparser.DumpAST.dot_ident_arg",false]],"dot_ident_arg() (celpy.evaluation.phase1transpiler method)":[[0,"celpy.evaluation.Phase1Transpiler.dot_ident_arg",false]],"doubletype (class in celpy.celtypes)":[[0,"celpy.celtypes.DoubleType",false]],"dumpast (class in celpy.celparser)":[[0,"celpy.celparser.DumpAST",false]],"durationtype (class in celpy.celtypes)":[[0,"celpy.celtypes.DurationType",false]],"encode() (celpy.adapter.celjsonencoder method)":[[0,"celpy.adapter.CELJSONEncoder.encode",false]],"environment (class in celpy.__init__)":[[0,"celpy.__init__.Environment",false]],"environment variable":[[0,"index-0",false]],"error_text() (celpy.celparser.celparser method)":[[0,"celpy.celparser.CELParser.error_text",false]],"eval_error() (in module celpy.evaluation)":[[0,"celpy.evaluation.eval_error",false]],"evaluate() (celpy.__init__.compiledrunner method)":[[0,"celpy.__init__.CompiledRunner.evaluate",false]],"evaluate() (celpy.__init__.interpretedrunner method)":[[0,"celpy.__init__.InterpretedRunner.evaluate",false]],"evaluate() (celpy.__init__.runner method)":[[0,"celpy.__init__.Runner.evaluate",false]],"evaluate() (celpy.c7nlib.c7n_interpreted_runner method)":[[0,"celpy.c7nlib.C7N_Interpreted_Runner.evaluate",false]],"evaluate() (celpy.evaluation.evaluator method)":[[0,"celpy.evaluation.Evaluator.evaluate",false]],"evaluate() (celpy.evaluation.transpiler method)":[[0,"celpy.evaluation.Transpiler.evaluate",false]],"evaluator (class in celpy.evaluation)":[[0,"celpy.evaluation.Evaluator",false]],"expr() (celpy.celparser.dumpast method)":[[0,"celpy.celparser.DumpAST.expr",false]],"expr() (celpy.evaluation.evaluator method)":[[0,"celpy.evaluation.Evaluator.expr",false]],"expr() (celpy.evaluation.phase1transpiler method)":[[0,"celpy.evaluation.Phase1Transpiler.expr",false]],"expr() (celpy.evaluation.phase2transpiler method)":[[0,"celpy.evaluation.Phase2Transpiler.expr",false]],"exprlist() (celpy.celparser.dumpast method)":[[0,"celpy.celparser.DumpAST.exprlist",false]],"exprlist() (celpy.evaluation.evaluator method)":[[0,"celpy.evaluation.Evaluator.exprlist",false]],"exprlist() (celpy.evaluation.phase1transpiler method)":[[0,"celpy.evaluation.Phase1Transpiler.exprlist",false]],"extended_name_path (celpy.evaluation.namecontainer attribute)":[[0,"celpy.evaluation.NameContainer.extended_name_path",false]],"fieldinits() (celpy.celparser.dumpast method)":[[0,"celpy.celparser.DumpAST.fieldinits",false]],"fieldinits() (celpy.evaluation.evaluator method)":[[0,"celpy.evaluation.Evaluator.fieldinits",false]],"fieldinits() (celpy.evaluation.phase1transpiler method)":[[0,"celpy.evaluation.Phase1Transpiler.fieldinits",false]],"find_name() (celpy.evaluation.namecontainer method)":[[0,"celpy.evaluation.NameContainer.find_name",false]],"flow_logs() (in module celpy.c7nlib)":[[0,"celpy.c7nlib.flow_logs",false]],"func_name() (celpy.evaluation.phase1transpiler method)":[[0,"celpy.evaluation.Phase1Transpiler.func_name",false]],"function_contains() (in module celpy.evaluation)":[[0,"celpy.evaluation.function_contains",false]],"function_endswith() (in module celpy.evaluation)":[[0,"celpy.evaluation.function_endsWith",false]],"function_eval() (celpy.evaluation.evaluator method)":[[0,"celpy.evaluation.Evaluator.function_eval",false]],"function_getdate() (in module celpy.evaluation)":[[0,"celpy.evaluation.function_getDate",false]],"function_getdayofmonth() (in module celpy.evaluation)":[[0,"celpy.evaluation.function_getDayOfMonth",false]],"function_getdayofweek() (in module celpy.evaluation)":[[0,"celpy.evaluation.function_getDayOfWeek",false]],"function_getdayofyear() (in module celpy.evaluation)":[[0,"celpy.evaluation.function_getDayOfYear",false]],"function_getfullyear() (in module celpy.evaluation)":[[0,"celpy.evaluation.function_getFullYear",false]],"function_gethours() (in module celpy.evaluation)":[[0,"celpy.evaluation.function_getHours",false]],"function_getmilliseconds() (in module celpy.evaluation)":[[0,"celpy.evaluation.function_getMilliseconds",false]],"function_getminutes() (in module celpy.evaluation)":[[0,"celpy.evaluation.function_getMinutes",false]],"function_getmonth() (in module celpy.evaluation)":[[0,"celpy.evaluation.function_getMonth",false]],"function_getseconds() (in module celpy.evaluation)":[[0,"celpy.evaluation.function_getSeconds",false]],"function_matches() (in module celpy.evaluation)":[[0,"celpy.evaluation.function_matches",false]],"function_size() (in module celpy.evaluation)":[[0,"celpy.evaluation.function_size",false]],"function_startswith() (in module celpy.evaluation)":[[0,"celpy.evaluation.function_startsWith",false]],"functiontype (class in celpy.celtypes)":[[0,"celpy.celtypes.FunctionType",false]],"get() (celpy.celtypes.maptype method)":[[0,"celpy.celtypes.MapType.get",false]],"get() (celpy.evaluation.activation method)":[[0,"celpy.evaluation.Activation.get",false]],"get() (celpy.evaluation.namecontainer method)":[[0,"celpy.evaluation.NameContainer.get",false]],"get_access_log() (in module celpy.c7nlib)":[[0,"celpy.c7nlib.get_access_log",false]],"get_accounts() (in module celpy.c7nlib)":[[0,"celpy.c7nlib.get_accounts",false]],"get_endpoints() (in module celpy.c7nlib)":[[0,"celpy.c7nlib.get_endpoints",false]],"get_health_events() (in module celpy.c7nlib)":[[0,"celpy.c7nlib.get_health_events",false]],"get_key_policy() (in module celpy.c7nlib)":[[0,"celpy.c7nlib.get_key_policy",false]],"get_load_balancer() (in module celpy.c7nlib)":[[0,"celpy.c7nlib.get_load_balancer",false]],"get_metrics() (in module celpy.c7nlib)":[[0,"celpy.c7nlib.get_metrics",false]],"get_options() (in module celpy.__main__)":[[0,"celpy.__main__.get_options",false]],"get_orgids() (in module celpy.c7nlib)":[[0,"celpy.c7nlib.get_orgids",false]],"get_protocols() (in module celpy.c7nlib)":[[0,"celpy.c7nlib.get_protocols",false]],"get_raw_health_events() (in module celpy.c7nlib)":[[0,"celpy.c7nlib.get_raw_health_events",false]],"get_raw_metrics() (in module celpy.c7nlib)":[[0,"celpy.c7nlib.get_raw_metrics",false]],"get_related_ids() (in module celpy.c7nlib)":[[0,"celpy.c7nlib.get_related_ids",false]],"get_related_igws() (in module celpy.c7nlib)":[[0,"celpy.c7nlib.get_related_igws",false]],"get_related_kms_keys() (in module celpy.c7nlib)":[[0,"celpy.c7nlib.get_related_kms_keys",false]],"get_related_nat_gateways() (in module celpy.c7nlib)":[[0,"celpy.c7nlib.get_related_nat_gateways",false]],"get_related_security_configs() (in module celpy.c7nlib)":[[0,"celpy.c7nlib.get_related_security_configs",false]],"get_related_sgs() (in module celpy.c7nlib)":[[0,"celpy.c7nlib.get_related_sgs",false]],"get_related_subnets() (in module celpy.c7nlib)":[[0,"celpy.c7nlib.get_related_subnets",false]],"get_related_vpc() (in module celpy.c7nlib)":[[0,"celpy.c7nlib.get_related_vpc",false]],"get_resource_policy() (in module celpy.c7nlib)":[[0,"celpy.c7nlib.get_resource_policy",false]],"get_vpces() (in module celpy.c7nlib)":[[0,"celpy.c7nlib.get_vpces",false]],"get_vpcs() (in module celpy.c7nlib)":[[0,"celpy.c7nlib.get_vpcs",false]],"getdate() (celpy.celtypes.timestamptype method)":[[0,"celpy.celtypes.TimestampType.getDate",false]],"getdayofmonth() (celpy.celtypes.timestamptype method)":[[0,"celpy.celtypes.TimestampType.getDayOfMonth",false]],"getdayofweek() (celpy.celtypes.timestamptype method)":[[0,"celpy.celtypes.TimestampType.getDayOfWeek",false]],"getdayofyear() (celpy.celtypes.timestamptype method)":[[0,"celpy.celtypes.TimestampType.getDayOfYear",false]],"getfullyear() (celpy.celtypes.timestamptype method)":[[0,"celpy.celtypes.TimestampType.getFullYear",false]],"gethours() (celpy.celtypes.durationtype method)":[[0,"celpy.celtypes.DurationType.getHours",false]],"gethours() (celpy.celtypes.timestamptype method)":[[0,"celpy.celtypes.TimestampType.getHours",false]],"getmilliseconds() (celpy.celtypes.durationtype method)":[[0,"celpy.celtypes.DurationType.getMilliseconds",false]],"getmilliseconds() (celpy.celtypes.timestamptype method)":[[0,"celpy.celtypes.TimestampType.getMilliseconds",false]],"getminutes() (celpy.celtypes.durationtype method)":[[0,"celpy.celtypes.DurationType.getMinutes",false]],"getminutes() (celpy.celtypes.timestamptype method)":[[0,"celpy.celtypes.TimestampType.getMinutes",false]],"getmonth() (celpy.celtypes.timestamptype method)":[[0,"celpy.celtypes.TimestampType.getMonth",false]],"getseconds() (celpy.celtypes.durationtype method)":[[0,"celpy.celtypes.DurationType.getSeconds",false]],"getseconds() (celpy.celtypes.timestamptype method)":[[0,"celpy.celtypes.TimestampType.getSeconds",false]],"glob() (in module celpy.c7nlib)":[[0,"celpy.c7nlib.glob",false]],"ident() (celpy.celparser.dumpast method)":[[0,"celpy.celparser.DumpAST.ident",false]],"ident() (celpy.evaluation.phase1transpiler method)":[[0,"celpy.evaluation.Phase1Transpiler.ident",false]],"ident_arg() (celpy.celparser.dumpast method)":[[0,"celpy.celparser.DumpAST.ident_arg",false]],"ident_arg() (celpy.evaluation.phase1transpiler method)":[[0,"celpy.evaluation.Phase1Transpiler.ident_arg",false]],"ident_arg() (celpy.evaluation.phase2transpiler method)":[[0,"celpy.evaluation.Phase2Transpiler.ident_arg",false]],"ident_pat (celpy.evaluation.namecontainer attribute)":[[0,"celpy.evaluation.NameContainer.ident_pat",false]],"ident_value() (celpy.evaluation.evaluator method)":[[0,"celpy.evaluation.Evaluator.ident_value",false]],"image() (in module celpy.c7nlib)":[[0,"celpy.c7nlib.image",false]],"int32value (class in celpy.__init__)":[[0,"celpy.__init__.Int32Value",false]],"int64() (in module celpy.celtypes)":[[0,"celpy.celtypes.int64",false]],"interpretedrunner (class in celpy.__init__)":[[0,"celpy.__init__.InterpretedRunner",false]],"intersect() (in module celpy.c7nlib)":[[0,"celpy.c7nlib.intersect",false]],"intro (celpy.__main__.cel_repl attribute)":[[0,"celpy.__main__.CEL_REPL.intro",false]],"inttype (class in celpy.celtypes)":[[0,"celpy.celtypes.IntType",false]],"ipv4network (class in celpy.c7nlib)":[[0,"celpy.c7nlib.IPv4Network",false]],"jmes_path() (in module celpy.c7nlib)":[[0,"celpy.c7nlib.jmes_path",false]],"jmes_path_map() (in module celpy.c7nlib)":[[0,"celpy.c7nlib.jmes_path_map",false]],"json_to_cel() (in module celpy.adapter)":[[0,"celpy.adapter.json_to_cel",false]],"key() (in module celpy.c7nlib)":[[0,"celpy.c7nlib.key",false]],"kms_alias() (in module celpy.c7nlib)":[[0,"celpy.c7nlib.kms_alias",false]],"kms_key() (in module celpy.c7nlib)":[[0,"celpy.c7nlib.kms_key",false]],"list_lit() (celpy.celparser.dumpast method)":[[0,"celpy.celparser.DumpAST.list_lit",false]],"list_lit() (celpy.evaluation.phase1transpiler method)":[[0,"celpy.evaluation.Phase1Transpiler.list_lit",false]],"listtype (class in celpy.celtypes)":[[0,"celpy.celtypes.ListType",false]],"literal() (celpy.celparser.dumpast method)":[[0,"celpy.celparser.DumpAST.literal",false]],"literal() (celpy.evaluation.evaluator method)":[[0,"celpy.evaluation.Evaluator.literal",false]],"literal() (celpy.evaluation.phase1transpiler method)":[[0,"celpy.evaluation.Phase1Transpiler.literal",false]],"load_annotations() (celpy.evaluation.namecontainer method)":[[0,"celpy.evaluation.NameContainer.load_annotations",false]],"load_values() (celpy.evaluation.namecontainer method)":[[0,"celpy.evaluation.NameContainer.load_values",false]],"logger (celpy.__main__.cel_repl attribute)":[[0,"celpy.__main__.CEL_REPL.logger",false]],"logger (celpy.evaluation.evaluator attribute)":[[0,"celpy.evaluation.Evaluator.logger",false]],"logger (celpy.evaluation.namecontainer attribute)":[[0,"celpy.evaluation.NameContainer.logger",false]],"logger (celpy.evaluation.transpiler attribute)":[[0,"celpy.evaluation.Transpiler.logger",false]],"logical_and() (in module celpy.celtypes)":[[0,"celpy.celtypes.logical_and",false]],"logical_condition() (in module celpy.celtypes)":[[0,"celpy.celtypes.logical_condition",false]],"logical_not() (in module celpy.celtypes)":[[0,"celpy.celtypes.logical_not",false]],"logical_or() (in module celpy.celtypes)":[[0,"celpy.celtypes.logical_or",false]],"macro_all() (in module celpy.evaluation)":[[0,"celpy.evaluation.macro_all",false]],"macro_exists() (in module celpy.evaluation)":[[0,"celpy.evaluation.macro_exists",false]],"macro_exists_one() (in module celpy.evaluation)":[[0,"celpy.evaluation.macro_exists_one",false]],"macro_filter() (in module celpy.evaluation)":[[0,"celpy.evaluation.macro_filter",false]],"macro_has_eval() (celpy.evaluation.evaluator method)":[[0,"celpy.evaluation.Evaluator.macro_has_eval",false]],"macro_map() (in module celpy.evaluation)":[[0,"celpy.evaluation.macro_map",false]],"main() (in module celpy.__main__)":[[0,"celpy.__main__.main",false]],"map_lit() (celpy.celparser.dumpast method)":[[0,"celpy.celparser.DumpAST.map_lit",false]],"map_lit() (celpy.evaluation.phase1transpiler method)":[[0,"celpy.evaluation.Phase1Transpiler.map_lit",false]],"mapinits() (celpy.celparser.dumpast method)":[[0,"celpy.celparser.DumpAST.mapinits",false]],"mapinits() (celpy.evaluation.evaluator method)":[[0,"celpy.evaluation.Evaluator.mapinits",false]],"mapinits() (celpy.evaluation.phase1transpiler method)":[[0,"celpy.evaluation.Phase1Transpiler.mapinits",false]],"maptype (class in celpy.celtypes)":[[0,"celpy.celtypes.MapType",false]],"marked_key() (in module celpy.c7nlib)":[[0,"celpy.c7nlib.marked_key",false]],"maxseconds (celpy.celtypes.durationtype attribute)":[[0,"celpy.celtypes.DurationType.MaxSeconds",false]],"member() (celpy.evaluation.evaluator method)":[[0,"celpy.evaluation.Evaluator.member",false]],"member() (celpy.evaluation.phase1transpiler method)":[[0,"celpy.evaluation.Phase1Transpiler.member",false]],"member_dot() (celpy.celparser.dumpast method)":[[0,"celpy.celparser.DumpAST.member_dot",false]],"member_dot() (celpy.evaluation.evaluator method)":[[0,"celpy.evaluation.Evaluator.member_dot",false]],"member_dot() (celpy.evaluation.phase1transpiler method)":[[0,"celpy.evaluation.Phase1Transpiler.member_dot",false]],"member_dot_arg() (celpy.celparser.dumpast method)":[[0,"celpy.celparser.DumpAST.member_dot_arg",false]],"member_dot_arg() (celpy.evaluation.evaluator method)":[[0,"celpy.evaluation.Evaluator.member_dot_arg",false]],"member_dot_arg() (celpy.evaluation.phase1transpiler method)":[[0,"celpy.evaluation.Phase1Transpiler.member_dot_arg",false]],"member_dot_arg() (celpy.evaluation.phase2transpiler method)":[[0,"celpy.evaluation.Phase2Transpiler.member_dot_arg",false]],"member_index() (celpy.celparser.dumpast method)":[[0,"celpy.celparser.DumpAST.member_index",false]],"member_index() (celpy.evaluation.evaluator method)":[[0,"celpy.evaluation.Evaluator.member_index",false]],"member_index() (celpy.evaluation.phase1transpiler method)":[[0,"celpy.evaluation.Phase1Transpiler.member_index",false]],"member_object() (celpy.celparser.dumpast method)":[[0,"celpy.celparser.DumpAST.member_object",false]],"member_object() (celpy.evaluation.evaluator method)":[[0,"celpy.evaluation.Evaluator.member_object",false]],"member_object() (celpy.evaluation.phase1transpiler method)":[[0,"celpy.evaluation.Phase1Transpiler.member_object",false]],"messagetype (class in celpy.celtypes)":[[0,"celpy.celtypes.MessageType",false]],"method_eval() (celpy.evaluation.evaluator method)":[[0,"celpy.evaluation.Evaluator.method_eval",false]],"minseconds (celpy.celtypes.durationtype attribute)":[[0,"celpy.celtypes.DurationType.MinSeconds",false]],"module":[[0,"module-celpy.__init__",false],[0,"module-celpy.__main__",false],[0,"module-celpy.adapter",false],[0,"module-celpy.c7nlib",false],[0,"module-celpy.celparser",false],[0,"module-celpy.celtypes",false],[0,"module-celpy.evaluation",false]],"multiplication() (celpy.celparser.dumpast method)":[[0,"celpy.celparser.DumpAST.multiplication",false]],"multiplication() (celpy.evaluation.evaluator method)":[[0,"celpy.evaluation.Evaluator.multiplication",false]],"multiplication() (celpy.evaluation.phase1transpiler method)":[[0,"celpy.evaluation.Phase1Transpiler.multiplication",false]],"multiplication_div() (celpy.celparser.dumpast method)":[[0,"celpy.celparser.DumpAST.multiplication_div",false]],"multiplication_mod() (celpy.celparser.dumpast method)":[[0,"celpy.celparser.DumpAST.multiplication_mod",false]],"multiplication_mul() (celpy.celparser.dumpast method)":[[0,"celpy.celparser.DumpAST.multiplication_mul",false]],"namecontainer (class in celpy.evaluation)":[[0,"celpy.evaluation.NameContainer",false]],"namecontainer.notfound":[[0,"celpy.evaluation.NameContainer.NotFound",false]],"nanosecondspersecond (celpy.celtypes.durationtype attribute)":[[0,"celpy.celtypes.DurationType.NanosecondsPerSecond",false]],"nested_activation() (celpy.evaluation.activation method)":[[0,"celpy.evaluation.Activation.nested_activation",false]],"new_activation() (celpy.__init__.runner method)":[[0,"celpy.__init__.Runner.new_activation",false]],"normalize() (in module celpy.c7nlib)":[[0,"celpy.c7nlib.normalize",false]],"nulltype (class in celpy.celtypes)":[[0,"celpy.celtypes.NullType",false]],"operator_in() (in module celpy.evaluation)":[[0,"celpy.evaluation.operator_in",false]],"packagetype (class in celpy.celtypes)":[[0,"celpy.celtypes.PackageType",false]],"paren_expr() (celpy.celparser.dumpast method)":[[0,"celpy.celparser.DumpAST.paren_expr",false]],"paren_expr() (celpy.evaluation.phase1transpiler method)":[[0,"celpy.evaluation.Phase1Transpiler.paren_expr",false]],"parent_iter() (celpy.evaluation.namecontainer method)":[[0,"celpy.evaluation.NameContainer.parent_iter",false]],"parse() (celpy.celparser.celparser method)":[[0,"celpy.celparser.CELParser.parse",false]],"parse_cidr() (in module celpy.c7nlib)":[[0,"celpy.c7nlib.parse_cidr",false]],"parse_text() (in module celpy.c7nlib)":[[0,"celpy.c7nlib.parse_text",false]],"phase1transpiler (class in celpy.evaluation)":[[0,"celpy.evaluation.Phase1Transpiler",false]],"phase2transpiler (class in celpy.evaluation)":[[0,"celpy.evaluation.Phase2Transpiler",false]],"preloop() (celpy.__main__.cel_repl method)":[[0,"celpy.__main__.CEL_REPL.preloop",false]],"present() (in module celpy.c7nlib)":[[0,"celpy.c7nlib.present",false]],"primary() (celpy.evaluation.evaluator method)":[[0,"celpy.evaluation.Evaluator.primary",false]],"primary() (celpy.evaluation.phase1transpiler method)":[[0,"celpy.evaluation.Phase1Transpiler.primary",false]],"process_json_doc() (in module celpy.__main__)":[[0,"celpy.__main__.process_json_doc",false]],"program() (celpy.__init__.environment method)":[[0,"celpy.__init__.Environment.program",false]],"prompt (celpy.__main__.cel_repl attribute)":[[0,"celpy.__main__.CEL_REPL.prompt",false]],"referent (class in celpy.evaluation)":[[0,"celpy.evaluation.Referent",false]],"relation() (celpy.celparser.dumpast method)":[[0,"celpy.celparser.DumpAST.relation",false]],"relation() (celpy.evaluation.evaluator method)":[[0,"celpy.evaluation.Evaluator.relation",false]],"relation() (celpy.evaluation.phase1transpiler method)":[[0,"celpy.evaluation.Phase1Transpiler.relation",false]],"relation_eq() (celpy.celparser.dumpast method)":[[0,"celpy.celparser.DumpAST.relation_eq",false]],"relation_ge() (celpy.celparser.dumpast method)":[[0,"celpy.celparser.DumpAST.relation_ge",false]],"relation_gt() (celpy.celparser.dumpast method)":[[0,"celpy.celparser.DumpAST.relation_gt",false]],"relation_in() (celpy.celparser.dumpast method)":[[0,"celpy.celparser.DumpAST.relation_in",false]],"relation_le() (celpy.celparser.dumpast method)":[[0,"celpy.celparser.DumpAST.relation_le",false]],"relation_lt() (celpy.celparser.dumpast method)":[[0,"celpy.celparser.DumpAST.relation_lt",false]],"relation_ne() (celpy.celparser.dumpast method)":[[0,"celpy.celparser.DumpAST.relation_ne",false]],"resolve_function() (celpy.evaluation.activation method)":[[0,"celpy.evaluation.Activation.resolve_function",false]],"resolve_name() (celpy.evaluation.namecontainer method)":[[0,"celpy.evaluation.NameContainer.resolve_name",false]],"resolve_variable() (celpy.evaluation.activation method)":[[0,"celpy.evaluation.Activation.resolve_variable",false]],"resource_schedule() (in module celpy.c7nlib)":[[0,"celpy.c7nlib.resource_schedule",false]],"result() (in module celpy.evaluation)":[[0,"celpy.evaluation.result",false]],"runner (class in celpy.__init__)":[[0,"celpy.__init__.Runner",false]],"scale (celpy.celtypes.durationtype attribute)":[[0,"celpy.celtypes.DurationType.scale",false]],"security_group() (in module celpy.c7nlib)":[[0,"celpy.c7nlib.security_group",false]],"set_activation() (celpy.evaluation.evaluator method)":[[0,"celpy.evaluation.Evaluator.set_activation",false]],"shield_protection() (in module celpy.c7nlib)":[[0,"celpy.c7nlib.shield_protection",false]],"shield_subscription() (in module celpy.c7nlib)":[[0,"celpy.c7nlib.shield_subscription",false]],"size_parse_cidr() (in module celpy.c7nlib)":[[0,"celpy.c7nlib.size_parse_cidr",false]],"stat() (in module celpy.__main__)":[[0,"celpy.__main__.stat",false]],"statements() (celpy.evaluation.phase2transpiler method)":[[0,"celpy.evaluation.Phase2Transpiler.statements",false]],"stringtype (class in celpy.celtypes)":[[0,"celpy.celtypes.StringType",false]],"sub_evaluator() (celpy.evaluation.evaluator method)":[[0,"celpy.evaluation.Evaluator.sub_evaluator",false]],"subnet() (in module celpy.c7nlib)":[[0,"celpy.c7nlib.subnet",false]],"subst() (in module celpy.c7nlib)":[[0,"celpy.c7nlib.subst",false]],"text_from() (in module celpy.c7nlib)":[[0,"celpy.c7nlib.text_from",false]],"the_activation (in module celpy.evaluation)":[[0,"celpy.evaluation.the_activation",false]],"timestamptype (class in celpy.celtypes)":[[0,"celpy.celtypes.TimestampType",false]],"to_python() (celpy.adapter.celjsonencoder static method)":[[0,"celpy.adapter.CELJSONEncoder.to_python",false]],"trace() (in module celpy.evaluation)":[[0,"celpy.evaluation.trace",false]],"transpile() (celpy.evaluation.transpiler method)":[[0,"celpy.evaluation.Transpiler.transpile",false]],"transpiler (class in celpy.evaluation)":[[0,"celpy.evaluation.Transpiler",false]],"transpilertree (class in celpy.evaluation)":[[0,"celpy.evaluation.TranspilerTree",false]],"tree_dump() (in module celpy.celparser)":[[0,"celpy.celparser.tree_dump",false]],"tree_node_class (celpy.__init__.compiledrunner attribute)":[[0,"celpy.__init__.CompiledRunner.tree_node_class",false]],"tree_node_class (celpy.__init__.runner attribute)":[[0,"celpy.__init__.Runner.tree_node_class",false]],"type_matched() (in module celpy.celtypes)":[[0,"celpy.celtypes.type_matched",false]],"typetype (class in celpy.celtypes)":[[0,"celpy.celtypes.TypeType",false]],"tz_aliases (celpy.celtypes.timestamptype attribute)":[[0,"celpy.celtypes.TimestampType.TZ_ALIASES",false]],"tz_name_lookup() (celpy.celtypes.timestamptype class method)":[[0,"celpy.celtypes.TimestampType.tz_name_lookup",false]],"tz_offset_parse() (celpy.celtypes.timestamptype class method)":[[0,"celpy.celtypes.TimestampType.tz_offset_parse",false]],"tz_parse() (celpy.celtypes.timestamptype static method)":[[0,"celpy.celtypes.TimestampType.tz_parse",false]],"uint64() (in module celpy.celtypes)":[[0,"celpy.celtypes.uint64",false]],"uinttype (class in celpy.celtypes)":[[0,"celpy.celtypes.UintType",false]],"unary() (celpy.celparser.dumpast method)":[[0,"celpy.celparser.DumpAST.unary",false]],"unary() (celpy.evaluation.evaluator method)":[[0,"celpy.evaluation.Evaluator.unary",false]],"unary() (celpy.evaluation.phase1transpiler method)":[[0,"celpy.evaluation.Phase1Transpiler.unary",false]],"unary_neg() (celpy.celparser.dumpast method)":[[0,"celpy.celparser.DumpAST.unary_neg",false]],"unary_not() (celpy.celparser.dumpast method)":[[0,"celpy.celparser.DumpAST.unary_not",false]],"unique_size() (in module celpy.c7nlib)":[[0,"celpy.c7nlib.unique_size",false]],"valid_key_type() (celpy.celtypes.maptype static method)":[[0,"celpy.celtypes.MapType.valid_key_type",false]],"value (celpy.evaluation.referent property)":[[0,"celpy.evaluation.Referent.value",false]],"value_from() (in module celpy.c7nlib)":[[0,"celpy.c7nlib.value_from",false]],"version() (in module celpy.c7nlib)":[[0,"celpy.c7nlib.version",false]],"visit() (celpy.evaluation.phase1transpiler method)":[[0,"celpy.evaluation.Phase1Transpiler.visit",false]],"visit_children() (celpy.evaluation.evaluator method)":[[0,"celpy.evaluation.Evaluator.visit_children",false]],"vpc() (in module celpy.c7nlib)":[[0,"celpy.c7nlib.vpc",false]],"web_acls() (in module celpy.c7nlib)":[[0,"celpy.c7nlib.web_acls",false]],"with_traceback() (celpy.evaluation.celevalerror method)":[[0,"celpy.evaluation.CELEvalError.with_traceback",false]]},"objects":{"celpy":[[0,0,0,"-","__init__"],[0,0,0,"-","__main__"],[0,0,0,"-","adapter"],[0,0,0,"-","c7nlib"],[0,0,0,"-","celparser"],[0,0,0,"-","celtypes"],[0,0,0,"-","evaluation"],[2,8,1,"cmdoption-celpy-a","--arg"],[2,8,1,"cmdoption-celpy-b","--boolean"],[2,8,1,"cmdoption-celpy-f","--format"],[2,8,1,"cmdoption-celpy-i","--interactive"],[2,8,1,"cmdoption-celpy-d","--json-document"],[2,8,1,"cmdoption-celpy-p","--json-package"],[2,8,1,"cmdoption-celpy-n","--null-input"],[2,8,1,"cmdoption-celpy-s","--slurp"],[2,8,1,"cmdoption-celpy-a","-a"],[2,8,1,"cmdoption-celpy-b","-b"],[2,8,1,"cmdoption-celpy-d","-d"],[2,8,1,"cmdoption-celpy-f","-f"],[2,8,1,"cmdoption-celpy-i","-i"],[2,8,1,"cmdoption-celpy-n","-n"],[2,8,1,"cmdoption-celpy-p","-p"],[2,8,1,"cmdoption-celpy-s","-s"],[2,8,1,"cmdoption-celpy-arg-expr","expr"]],"celpy.__init__":[[0,1,1,"","CompiledRunner"],[0,1,1,"","Environment"],[0,1,1,"","Int32Value"],[0,1,1,"","InterpretedRunner"],[0,1,1,"","Runner"]],"celpy.__init__.CompiledRunner":[[0,2,1,"","__abstractmethods__"],[0,3,1,"","__init__"],[0,3,1,"","evaluate"],[0,2,1,"","tree_node_class"]],"celpy.__init__.Environment":[[0,3,1,"","__init__"],[0,3,1,"","__repr__"],[0,3,1,"","compile"],[0,3,1,"","program"]],"celpy.__init__.Int32Value":[[0,3,1,"","__new__"]],"celpy.__init__.InterpretedRunner":[[0,2,1,"","__abstractmethods__"],[0,3,1,"","evaluate"]],"celpy.__init__.Runner":[[0,2,1,"","__abstractmethods__"],[0,3,1,"","__init__"],[0,3,1,"","__repr__"],[0,3,1,"","evaluate"],[0,3,1,"","new_activation"],[0,2,1,"","tree_node_class"]],"celpy.__main__":[[0,1,1,"","CEL_REPL"],[0,4,1,"","arg_type_value"],[0,4,1,"","get_options"],[0,4,1,"","main"],[0,4,1,"","process_json_doc"],[0,4,1,"","stat"]],"celpy.__main__.CEL_REPL":[[0,3,1,"","cel_eval"],[0,3,1,"","default"],[0,3,1,"","do_EOF"],[0,3,1,"","do_bye"],[0,3,1,"","do_exit"],[0,3,1,"","do_quit"],[0,3,1,"","do_set"],[0,3,1,"","do_show"],[0,2,1,"","intro"],[0,2,1,"","logger"],[0,3,1,"","preloop"],[0,2,1,"","prompt"]],"celpy.adapter":[[0,1,1,"","CELJSONDecoder"],[0,1,1,"","CELJSONEncoder"],[0,4,1,"","json_to_cel"]],"celpy.adapter.CELJSONDecoder":[[0,3,1,"","decode"]],"celpy.adapter.CELJSONEncoder":[[0,3,1,"","default"],[0,3,1,"","encode"],[0,3,1,"","to_python"]],"celpy.c7nlib":[[0,1,1,"","C7NContext"],[0,1,1,"","C7N_Interpreted_Runner"],[0,1,1,"","ComparableVersion"],[0,1,1,"","IPv4Network"],[0,4,1,"","absent"],[0,4,1,"","all_dbsubenet_groups"],[0,4,1,"","all_images"],[0,4,1,"","all_instance_profiles"],[0,4,1,"","all_launch_configuration_names"],[0,4,1,"","all_scan_groups"],[0,4,1,"","all_service_roles"],[0,4,1,"","all_snapshots"],[0,4,1,"","arn_split"],[0,4,1,"","credentials"],[0,4,1,"","describe_db_snapshot_attributes"],[0,4,1,"","describe_subscription_filters"],[0,4,1,"","difference"],[0,4,1,"","flow_logs"],[0,4,1,"","get_access_log"],[0,4,1,"","get_accounts"],[0,4,1,"","get_endpoints"],[0,4,1,"","get_health_events"],[0,4,1,"","get_key_policy"],[0,4,1,"","get_load_balancer"],[0,4,1,"","get_metrics"],[0,4,1,"","get_orgids"],[0,4,1,"","get_protocols"],[0,4,1,"","get_raw_health_events"],[0,4,1,"","get_raw_metrics"],[0,4,1,"","get_related_ids"],[0,4,1,"","get_related_igws"],[0,4,1,"","get_related_kms_keys"],[0,4,1,"","get_related_nat_gateways"],[0,4,1,"","get_related_security_configs"],[0,4,1,"","get_related_sgs"],[0,4,1,"","get_related_subnets"],[0,4,1,"","get_related_vpc"],[0,4,1,"","get_resource_policy"],[0,4,1,"","get_vpces"],[0,4,1,"","get_vpcs"],[0,4,1,"","glob"],[0,4,1,"","image"],[0,4,1,"","intersect"],[0,4,1,"","jmes_path"],[0,4,1,"","jmes_path_map"],[0,4,1,"","key"],[0,4,1,"","kms_alias"],[0,4,1,"","kms_key"],[0,4,1,"","marked_key"],[0,4,1,"","normalize"],[0,4,1,"","parse_cidr"],[0,4,1,"","parse_text"],[0,4,1,"","present"],[0,4,1,"","resource_schedule"],[0,4,1,"","security_group"],[0,4,1,"","shield_protection"],[0,4,1,"","shield_subscription"],[0,4,1,"","size_parse_cidr"],[0,4,1,"","subnet"],[0,4,1,"","subst"],[0,4,1,"","text_from"],[0,4,1,"","unique_size"],[0,4,1,"","value_from"],[0,4,1,"","version"],[0,4,1,"","vpc"],[0,4,1,"","web_acls"]],"celpy.c7nlib.C7NContext":[[0,3,1,"","__enter__"],[0,3,1,"","__exit__"],[0,3,1,"","__init__"],[0,3,1,"","__repr__"]],"celpy.c7nlib.C7N_Interpreted_Runner":[[0,2,1,"","__abstractmethods__"],[0,3,1,"","evaluate"]],"celpy.c7nlib.ComparableVersion":[[0,3,1,"","__eq__"],[0,2,1,"","__hash__"]],"celpy.c7nlib.IPv4Network":[[0,3,1,"","__contains__"],[0,3,1,"","contains"]],"celpy.celparser":[[0,5,1,"","CELParseError"],[0,1,1,"","CELParser"],[0,1,1,"","DumpAST"],[0,4,1,"","tree_dump"]],"celpy.celparser.CELParseError":[[0,3,1,"","__init__"]],"celpy.celparser.CELParser":[[0,2,1,"","CEL_PARSER"],[0,3,1,"","__init__"],[0,3,1,"","ambiguous_literals"],[0,3,1,"","error_text"],[0,3,1,"","parse"]],"celpy.celparser.DumpAST":[[0,3,1,"","__init__"],[0,2,1,"","__parameters__"],[0,3,1,"","addition"],[0,3,1,"","addition_add"],[0,3,1,"","addition_sub"],[0,3,1,"","conditionaland"],[0,3,1,"","conditionalor"],[0,3,1,"","display"],[0,3,1,"","dot_ident"],[0,3,1,"","dot_ident_arg"],[0,3,1,"","expr"],[0,3,1,"","exprlist"],[0,3,1,"","fieldinits"],[0,3,1,"","ident"],[0,3,1,"","ident_arg"],[0,3,1,"","list_lit"],[0,3,1,"","literal"],[0,3,1,"","map_lit"],[0,3,1,"","mapinits"],[0,3,1,"","member_dot"],[0,3,1,"","member_dot_arg"],[0,3,1,"","member_index"],[0,3,1,"","member_object"],[0,3,1,"","multiplication"],[0,3,1,"","multiplication_div"],[0,3,1,"","multiplication_mod"],[0,3,1,"","multiplication_mul"],[0,3,1,"","paren_expr"],[0,3,1,"","relation"],[0,3,1,"","relation_eq"],[0,3,1,"","relation_ge"],[0,3,1,"","relation_gt"],[0,3,1,"","relation_in"],[0,3,1,"","relation_le"],[0,3,1,"","relation_lt"],[0,3,1,"","relation_ne"],[0,3,1,"","unary"],[0,3,1,"","unary_neg"],[0,3,1,"","unary_not"]],"celpy.celtypes":[[0,1,1,"","BoolType"],[0,1,1,"","BytesType"],[0,1,1,"","DoubleType"],[0,1,1,"","DurationType"],[0,1,1,"","FunctionType"],[0,1,1,"","IntType"],[0,1,1,"","ListType"],[0,1,1,"","MapType"],[0,1,1,"","MessageType"],[0,1,1,"","NullType"],[0,1,1,"","PackageType"],[0,1,1,"","StringType"],[0,1,1,"","TimestampType"],[0,1,1,"","TypeType"],[0,1,1,"","UintType"],[0,4,1,"","int64"],[0,4,1,"","logical_and"],[0,4,1,"","logical_condition"],[0,4,1,"","logical_not"],[0,4,1,"","logical_or"],[0,4,1,"","type_matched"],[0,4,1,"","uint64"]],"celpy.celtypes.BoolType":[[0,3,1,"","__hash__"],[0,3,1,"","__neg__"],[0,3,1,"","__new__"],[0,3,1,"","__repr__"],[0,3,1,"","__str__"]],"celpy.celtypes.BytesType":[[0,3,1,"","__new__"],[0,3,1,"","__repr__"]],"celpy.celtypes.DoubleType":[[0,3,1,"","__eq__"],[0,3,1,"","__hash__"],[0,3,1,"","__mod__"],[0,3,1,"","__ne__"],[0,3,1,"","__neg__"],[0,3,1,"","__new__"],[0,3,1,"","__repr__"],[0,3,1,"","__rmod__"],[0,3,1,"","__rtruediv__"],[0,3,1,"","__str__"],[0,3,1,"","__truediv__"]],"celpy.celtypes.DurationType":[[0,2,1,"","MaxSeconds"],[0,2,1,"","MinSeconds"],[0,2,1,"","NanosecondsPerSecond"],[0,3,1,"","__add__"],[0,3,1,"","__new__"],[0,3,1,"","__radd__"],[0,3,1,"","__repr__"],[0,3,1,"","__str__"],[0,3,1,"","getHours"],[0,3,1,"","getMilliseconds"],[0,3,1,"","getMinutes"],[0,3,1,"","getSeconds"],[0,2,1,"","scale"]],"celpy.celtypes.FunctionType":[[0,3,1,"","__call__"]],"celpy.celtypes.IntType":[[0,3,1,"","__add__"],[0,3,1,"","__eq__"],[0,3,1,"","__floordiv__"],[0,3,1,"","__ge__"],[0,3,1,"","__gt__"],[0,3,1,"","__hash__"],[0,3,1,"","__le__"],[0,3,1,"","__lt__"],[0,3,1,"","__mod__"],[0,3,1,"","__mul__"],[0,3,1,"","__ne__"],[0,3,1,"","__neg__"],[0,3,1,"","__new__"],[0,3,1,"","__radd__"],[0,3,1,"","__repr__"],[0,3,1,"","__rfloordiv__"],[0,3,1,"","__rmod__"],[0,3,1,"","__rmul__"],[0,3,1,"","__rsub__"],[0,3,1,"","__rtruediv__"],[0,3,1,"","__str__"],[0,3,1,"","__sub__"],[0,3,1,"","__truediv__"]],"celpy.celtypes.ListType":[[0,3,1,"","__eq__"],[0,3,1,"","__ge__"],[0,3,1,"","__gt__"],[0,2,1,"","__hash__"],[0,3,1,"","__le__"],[0,3,1,"","__lt__"],[0,3,1,"","__ne__"],[0,2,1,"","__orig_bases__"],[0,2,1,"","__parameters__"],[0,3,1,"","__repr__"],[0,3,1,"","contains"]],"celpy.celtypes.MapType":[[0,3,1,"","__eq__"],[0,3,1,"","__getitem__"],[0,2,1,"","__hash__"],[0,3,1,"","__init__"],[0,3,1,"","__ne__"],[0,2,1,"","__orig_bases__"],[0,2,1,"","__parameters__"],[0,3,1,"","__repr__"],[0,3,1,"","contains"],[0,3,1,"","get"],[0,3,1,"","valid_key_type"]],"celpy.celtypes.MessageType":[[0,3,1,"","__init__"],[0,2,1,"","__parameters__"]],"celpy.celtypes.NullType":[[0,3,1,"","__eq__"],[0,2,1,"","__hash__"],[0,3,1,"","__ne__"]],"celpy.celtypes.PackageType":[[0,2,1,"","__parameters__"]],"celpy.celtypes.StringType":[[0,3,1,"","__eq__"],[0,3,1,"","__hash__"],[0,3,1,"","__ne__"],[0,3,1,"","__new__"],[0,3,1,"","__repr__"],[0,3,1,"","contains"]],"celpy.celtypes.TimestampType":[[0,2,1,"","TZ_ALIASES"],[0,3,1,"","__add__"],[0,3,1,"","__new__"],[0,3,1,"","__radd__"],[0,3,1,"","__repr__"],[0,3,1,"","__str__"],[0,3,1,"","__sub__"],[0,3,1,"","getDate"],[0,3,1,"","getDayOfMonth"],[0,3,1,"","getDayOfWeek"],[0,3,1,"","getDayOfYear"],[0,3,1,"","getFullYear"],[0,3,1,"","getHours"],[0,3,1,"","getMilliseconds"],[0,3,1,"","getMinutes"],[0,3,1,"","getMonth"],[0,3,1,"","getSeconds"],[0,3,1,"","tz_name_lookup"],[0,3,1,"","tz_offset_parse"],[0,3,1,"","tz_parse"]],"celpy.celtypes.TypeType":[[0,3,1,"","__new__"]],"celpy.celtypes.UintType":[[0,3,1,"","__add__"],[0,3,1,"","__eq__"],[0,3,1,"","__floordiv__"],[0,3,1,"","__hash__"],[0,3,1,"","__mod__"],[0,3,1,"","__mul__"],[0,3,1,"","__ne__"],[0,3,1,"","__neg__"],[0,3,1,"","__new__"],[0,3,1,"","__radd__"],[0,3,1,"","__repr__"],[0,3,1,"","__rfloordiv__"],[0,3,1,"","__rmod__"],[0,3,1,"","__rmul__"],[0,3,1,"","__rsub__"],[0,3,1,"","__rtruediv__"],[0,3,1,"","__str__"],[0,3,1,"","__sub__"],[0,3,1,"","__truediv__"]],"celpy.evaluation":[[0,1,1,"","Activation"],[0,5,1,"","CELEvalError"],[0,5,1,"","CELSyntaxError"],[0,5,1,"","CELUnsupportedError"],[0,1,1,"","Evaluator"],[0,1,1,"","NameContainer"],[0,1,1,"","Phase1Transpiler"],[0,1,1,"","Phase2Transpiler"],[0,1,1,"","Referent"],[0,1,1,"","Transpiler"],[0,1,1,"","TranspilerTree"],[0,4,1,"","bool_eq"],[0,4,1,"","bool_ge"],[0,4,1,"","bool_gt"],[0,4,1,"","bool_le"],[0,4,1,"","bool_lt"],[0,4,1,"","bool_ne"],[0,4,1,"","boolean"],[0,4,1,"","celbytes"],[0,4,1,"","celstr"],[0,4,1,"","eval_error"],[0,4,1,"","function_contains"],[0,4,1,"","function_endsWith"],[0,4,1,"","function_getDate"],[0,4,1,"","function_getDayOfMonth"],[0,4,1,"","function_getDayOfWeek"],[0,4,1,"","function_getDayOfYear"],[0,4,1,"","function_getFullYear"],[0,4,1,"","function_getHours"],[0,4,1,"","function_getMilliseconds"],[0,4,1,"","function_getMinutes"],[0,4,1,"","function_getMonth"],[0,4,1,"","function_getSeconds"],[0,4,1,"","function_matches"],[0,4,1,"","function_size"],[0,4,1,"","function_startsWith"],[0,4,1,"","macro_all"],[0,4,1,"","macro_exists"],[0,4,1,"","macro_exists_one"],[0,4,1,"","macro_filter"],[0,4,1,"","macro_map"],[0,4,1,"","operator_in"],[0,4,1,"","result"],[0,7,1,"","the_activation"],[0,4,1,"","trace"]],"celpy.evaluation.Activation":[[0,3,1,"","__getattr__"],[0,3,1,"","__init__"],[0,3,1,"","__repr__"],[0,3,1,"","clone"],[0,3,1,"","get"],[0,3,1,"","nested_activation"],[0,3,1,"","resolve_function"],[0,3,1,"","resolve_variable"]],"celpy.evaluation.CELEvalError":[[0,3,1,"","__add__"],[0,3,1,"","__call__"],[0,3,1,"","__eq__"],[0,3,1,"","__floordiv__"],[0,2,1,"","__hash__"],[0,3,1,"","__init__"],[0,3,1,"","__mod__"],[0,3,1,"","__mul__"],[0,3,1,"","__neg__"],[0,3,1,"","__pow__"],[0,3,1,"","__radd__"],[0,3,1,"","__repr__"],[0,3,1,"","__rfloordiv__"],[0,3,1,"","__rmod__"],[0,3,1,"","__rmul__"],[0,3,1,"","__rpow__"],[0,3,1,"","__rsub__"],[0,3,1,"","__rtruediv__"],[0,3,1,"","__sub__"],[0,3,1,"","__truediv__"],[0,3,1,"","with_traceback"]],"celpy.evaluation.CELSyntaxError":[[0,3,1,"","__init__"]],"celpy.evaluation.CELUnsupportedError":[[0,3,1,"","__init__"]],"celpy.evaluation.Evaluator":[[0,2,1,"","__abstractmethods__"],[0,3,1,"","__init__"],[0,2,1,"","__parameters__"],[0,3,1,"","addition"],[0,3,1,"","build_macro_eval"],[0,3,1,"","build_reduce_macro_eval"],[0,3,1,"","build_ss_macro_eval"],[0,3,1,"","conditionaland"],[0,3,1,"","conditionalor"],[0,3,1,"","evaluate"],[0,3,1,"","expr"],[0,3,1,"","exprlist"],[0,3,1,"","fieldinits"],[0,3,1,"","function_eval"],[0,3,1,"","ident_value"],[0,3,1,"","literal"],[0,2,1,"","logger"],[0,3,1,"","macro_has_eval"],[0,3,1,"","mapinits"],[0,3,1,"","member"],[0,3,1,"","member_dot"],[0,3,1,"","member_dot_arg"],[0,3,1,"","member_index"],[0,3,1,"","member_object"],[0,3,1,"","method_eval"],[0,3,1,"","multiplication"],[0,3,1,"","primary"],[0,3,1,"","relation"],[0,3,1,"","set_activation"],[0,3,1,"","sub_evaluator"],[0,3,1,"","unary"],[0,3,1,"","visit_children"]],"celpy.evaluation.NameContainer":[[0,5,1,"","NotFound"],[0,3,1,"","__init__"],[0,2,1,"","__orig_bases__"],[0,2,1,"","__parameters__"],[0,3,1,"","__repr__"],[0,3,1,"","clone"],[0,3,1,"","dict_find_name"],[0,2,1,"","extended_name_path"],[0,3,1,"","find_name"],[0,3,1,"","get"],[0,2,1,"","ident_pat"],[0,3,1,"","load_annotations"],[0,3,1,"","load_values"],[0,2,1,"","logger"],[0,3,1,"","parent_iter"],[0,3,1,"","resolve_name"]],"celpy.evaluation.Phase1Transpiler":[[0,3,1,"","__init__"],[0,2,1,"","__parameters__"],[0,3,1,"","addition"],[0,3,1,"","conditionaland"],[0,3,1,"","conditionalor"],[0,3,1,"","dot_ident"],[0,3,1,"","dot_ident_arg"],[0,3,1,"","expr"],[0,3,1,"","exprlist"],[0,3,1,"","fieldinits"],[0,3,1,"","func_name"],[0,3,1,"","ident"],[0,3,1,"","ident_arg"],[0,3,1,"","list_lit"],[0,3,1,"","literal"],[0,3,1,"","map_lit"],[0,3,1,"","mapinits"],[0,3,1,"","member"],[0,3,1,"","member_dot"],[0,3,1,"","member_dot_arg"],[0,3,1,"","member_index"],[0,3,1,"","member_object"],[0,3,1,"","multiplication"],[0,3,1,"","paren_expr"],[0,3,1,"","primary"],[0,3,1,"","relation"],[0,3,1,"","unary"],[0,3,1,"","visit"]],"celpy.evaluation.Phase2Transpiler":[[0,3,1,"","__init__"],[0,2,1,"","__parameters__"],[0,3,1,"","conditionaland"],[0,3,1,"","conditionalor"],[0,3,1,"","expr"],[0,3,1,"","ident_arg"],[0,3,1,"","member_dot_arg"],[0,3,1,"","statements"]],"celpy.evaluation.Referent":[[0,3,1,"","__eq__"],[0,2,1,"","__hash__"],[0,3,1,"","__init__"],[0,3,1,"","__repr__"],[0,3,1,"","clone"],[0,6,1,"","value"]],"celpy.evaluation.Transpiler":[[0,3,1,"","__init__"],[0,3,1,"","evaluate"],[0,2,1,"","logger"],[0,3,1,"","transpile"]],"celpy.evaluation.TranspilerTree":[[0,3,1,"","__init__"],[0,2,1,"","__parameters__"],[0,2,1,"","children"],[0,2,1,"","data"]],"features.steps":[[4,0,0,"-","c7n_integration"],[4,0,0,"-","cli_binding"],[4,0,0,"-","integration_binding"]],"python-tools/pb2g.py-[-g-docker|local]-[-o-output]-[-sv]-source":[[4,8,1,"cmdoption-python-tools-pb2g.py-g-docker-local-o-output-sv-source-g","--gherkinizer"],[4,8,1,"cmdoption-python-tools-pb2g.py-g-docker-local-o-output-sv-source-o","--output"],[4,8,1,"cmdoption-python-tools-pb2g.py-g-docker-local-o-output-sv-source-s","--silent"],[4,8,1,"cmdoption-python-tools-pb2g.py-g-docker-local-o-output-sv-source-v","--verbose"],[4,8,1,"cmdoption-python-tools-pb2g.py-g-docker-local-o-output-sv-source-g","-g"],[4,8,1,"cmdoption-python-tools-pb2g.py-g-docker-local-o-output-sv-source-o","-o"],[4,8,1,"cmdoption-python-tools-pb2g.py-g-docker-local-o-output-sv-source-s","-s"],[4,8,1,"cmdoption-python-tools-pb2g.py-g-docker-local-o-output-sv-source-v","-v"],[4,8,1,"cmdoption-python-tools-pb2g.py-g-docker-local-o-output-sv-source-arg-source","source"]]},"objnames":{"0":["py","module","Python module"],"1":["py","class","Python class"],"2":["py","attribute","Python attribute"],"3":["py","method","Python method"],"4":["py","function","Python function"],"5":["py","exception","Python exception"],"6":["py","property","Python property"],"7":["py","data","Python data"],"8":["std","cmdoption","program option"]},"objtypes":{"0":"py:module","1":"py:class","2":"py:attribute","3":"py:method","4":"py:function","5":"py:exception","6":"py:property","7":"py:data","8":"std:cmdoption"},"terms":{"":[0,1,2,3,4,7,8],"0":[0,1,2,4,7],"00":7,"001":0,"002":1,"01":1,"011":1,"01ab23cd":1,"02":[0,1],"03":1,"03t16":7,"03z":[2,7],"04":[1,7],"05":1,"06":0,"06t05":7,"06t20":2,"07":[1,2,7],"08":[1,7],"084":1,"09":0,"0u":4,"1":[0,1,2,3,7],"10":[0,1],"100":[1,2],"1000":1,"10000":1,"1000000000":0,"1004":1,"101":1,"1021":1,"1023":1,"10240":1,"1026":1,"103":1,"1034":1,"104":1,"1041":1,"1045":1,"105":1,"1057":1,"106":1,"1066":1,"108":1,"1080":1,"10am":1,"11":[1,2],"111":1,"1124":1,"113":2,"1144":1,"115":1,"116":1,"117":1,"1178":1,"1180":1,"1191":1,"12":[1,4],"121":1,"1211":1,"122":1,"123":[0,1,7],"1232":1,"1234567890":0,"123u":0,"125":1,"127":[0,1],"128":1,"1295":1,"12ab34cd":1,"13":[2,6],"131":1,"132":1,"1333":1,"134":1,"135":2,"136":1,"138":0,"13t23":0,"14":[0,1],"140":7,"1410":1,"14159":[],"1415929203539825":2,"1437":1,"145":1,"147":1,"1474":1,"149":0,"1493":1,"15":[1,7],"1505":1,"1513":1,"154":1,"1584":1,"16":1,"161":1,"1613":1,"162":1,"167":1,"16777234":2,"1686":1,"17":1,"1704":1,"171":1,"1728":1,"173":1,"1779":1,"178":1,"1787":1,"179":1,"1793":1,"1799":1,"18":[0,1],"182":1,"183":1,"184":1,"18446744073709551615":0,"185":1,"189":1,"19":1,"190":1,"194":1,"1954":1,"196":1,"197":1,"198":1,"1e":0,"1st":1,"1u":0,"2":[0,1,2],"20":1,"2006":2,"2009":0,"2012":1,"2016":1,"2017":1,"2018":7,"2019":1,"2020":1,"2021":1,"2025":2,"203":1,"2048":1,"205":1,"206":1,"20z":2,"21":[0,1,7],"212":1,"21d":1,"21z":2,"22":1,"223":1,"2242":1,"228":1,"2292":1,"23":[1,7],"232":1,"237":1,"238":1,"24":1,"24h":7,"24x5":1,"25":1,"250":1,"2500000":1,"256":0,"257":1,"26":1,"262":1,"268":1,"27":[1,2,7],"273":1,"277":[0,1],"2788":1,"27t18":2,"28":1,"285":0,"29":1,"293":1,"299":1,"2h45m":0,"2nd":[],"3":[0,1,2,4,6,7],"30":[0,1,2],"300m":0,"301":1,"305":1,"30z":0,"31":[0,2],"313":1,"314":1,"315576000000":0,"317":1,"318":1,"32":1,"325":1,"329":1,"34":1,"341":1,"341035":2,"345":1,"348":1,"35":1,"355":[0,2],"35dai":1,"36":1,"3600":0,"364":1,"37":1,"372":1,"373":1,"390":1,"3a":1,"4":[0,1],"40":[1,7],"400":1,"402":1,"408":1,"409":1,"4096":2,"41":[1,2],"412":1,"413":1,"42":[0,1,2,8],"424":1,"429":1,"42u":0,"43":1,"431":1,"4320":2,"432u":0,"436":1,"438":1,"443":1,"448":1,"45":1,"453":1,"455":1,"456":0,"47":[1,7],"48":1,"485":1,"490":1,"493":1,"4d":0,"5":[0,1],"50":1,"500":7,"5000":1,"505":1,"5103":1,"512":1,"5120":1,"514":1,"516":1,"52":1,"54":1,"548":1,"549":1,"55":1,"556":1,"562":1,"563":1,"567":1,"569":1,"57":1,"575":1,"5759":7,"58":1,"592":1,"597":1,"5h":0,"6":[0,1,2],"60":[0,1],"600":7,"604":1,"608":1,"610":1,"612000":1,"615":1,"619":1,"622":1,"628":1,"63":0,"64":[1,7,8],"643":1,"644":1,"65":1,"656":1,"659":1,"66":1,"662":1,"666":1,"667":1,"676":1,"69":1,"692":1,"6am":1,"6f":2,"6hr":1,"6pm":1,"7":[0,1,2],"70":0,"711":1,"720":1,"729":1,"739":1,"74":1,"744":1,"748":1,"758":1,"767":1,"78":1,"786":1,"791":1,"7am":1,"7pm":1,"8":[1,7],"80":1,"806":1,"8098":1,"81":1,"812":1,"826":1,"829":1,"830985915492956":2,"842":1,"845":1,"86":1,"86400":[0,1],"87":1,"88":1,"8x5":1,"9":[0,1],"90":1,"92":1,"9223372036854775807":0,"9223372036854775808":0,"93":1,"932":1,"934":1,"94":1,"97":1,"98":1,"987":1,"988":1,"99":1,"9pm":1,"A":[0,1,2,4,7,8],"AND":0,"And":[0,2],"As":[2,7,8],"At":[0,8],"But":0,"By":[0,2],"For":[0,1,2,4,6,7,8],"If":[0,1,2,3,4,7,8],"In":[0,1,2,3,6,7,8],"It":[0,1,2,4,7,8],"NOT":[0,1],"No":[0,1,4,7,8],"Not":[0,1],"Of":1,"On":[1,7],"One":[0,1],"Or":0,"Such":8,"THe":2,"That":0,"The":[1,2,3,5,6,7],"Then":[0,1,4],"There":[0,1,2,4,5,7,8],"These":[0,1,4,7,8],"To":[0,3,7],"With":[0,1],"_":[0,7],"__abstractmethods__":0,"__add__":0,"__call__":[0,7],"__contains__":0,"__enter__":0,"__eq__":0,"__exit__":0,"__floordiv__":0,"__ge__":0,"__getattr__":0,"__getitem__":0,"__globals__":[],"__gt__":0,"__hash__":0,"__init__":[0,1,7,8],"__isinstance__":0,"__le__":0,"__lt__":0,"__main__":[5,8],"__mod__":0,"__mul__":0,"__ne__":0,"__neg__":0,"__new__":0,"__orig_bases__":0,"__parameters__":0,"__pow__":0,"__radd__":0,"__repr__":0,"__rfloordiv__":0,"__rmod__":0,"__rmul__":0,"__rpow__":0,"__rsub__":0,"__rtruediv__":0,"__str__":0,"__sub__":0,"__traceback__":0,"__truediv__":0,"_a":0,"_init":[],"_leaf_t":[],"_oper":0,"_pull_ami_snapshot":[0,1],"_pull_asg_imag":[0,1],"_pull_asg_snapshot":[0,1],"_pull_ec2_imag":[0,1],"_w":0,"ab":0,"abandond":1,"abc":[0,1],"abil":2,"abl":[0,1],"about":[1,5,7],"abov":[0,1,7],"absenc":1,"absent":[0,1],"absolut":0,"abstract":[0,7],"abstractmethod":0,"accept":[0,1],"access":0,"access_kei":1,"accesskeysperuserquota":1,"accesslog":1,"account":[0,7],"account_id":[0,1],"account_shield_subscript":[0,1],"accountaccesskeyspres":1,"accountcredentialreport":1,"accountmfaen":1,"accountnumb":1,"accountpasswordpolici":1,"accounts_url":1,"accountsigningcertificatespres":1,"achiev":2,"acl":1,"aclawss3cidr":1,"aclsubnetfilt":1,"acm":[1,7],"acquir":1,"across":[1,5],"act":1,"action":[0,1,7],"action_d":[0,1],"activ":[0,1,2,7,8],"actual":[0,7],"ad":[0,1],"adapt":[5,7,8],"add":[0,1,3,6,8],"addit":[1,7,8],"addition":[0,1,2,7,8],"addition_add":0,"addition_sub":0,"additionalproperti":1,"addo":0,"addr":1,"address":[0,1],"adher":1,"admin":1,"administratoraccess":1,"advanc":1,"advantag":2,"advisor":1,"aes128":1,"aes256":1,"after":[0,1,4],"ag":7,"again":[0,7],"against":[0,1],"agefilt":1,"agre":0,"ahom":2,"akward":0,"alarm":1,"alb":1,"alblog":1,"alert":1,"alia":0,"alias":[0,1],"aliasnam":1,"all":[0,2,3,4,7,8],"all_dbsubenet_group":[0,1],"all_imag":[0,1],"all_instance_profil":[0,1],"all_launch_configuration_nam":[0,1],"all_scan_group":[0,1],"all_service_rol":[0,1],"all_snapshot":[0,1],"all_valu":1,"alloc":1,"allocatedstorag":1,"allow":[0,2,7,8],"allow_nan":0,"allow_websit":1,"allowalliampolici":1,"almost":0,"alon":[1,8],"along":[0,1],"alreadi":0,"also":[0,1,2,7,8],"altern":[0,1],"alwai":[0,1,8],"amazon":1,"amazonaw":1,"amazons3":1,"ambigu":0,"ambiguous_liter":0,"america":1,"ami":0,"amicrossaccountfilt":1,"among":[0,1],"amort":8,"an":[0,1,2,4,5,7,8],"analysi":1,"analyt":1,"ancient":1,"ani":[0,1,2,3,4,7,8],"annot":[0,1,7,8],"anoth":[0,1,2,5,7,8],"anyof":1,"ap":1,"api":[1,5,8],"apigw":1,"apikeyrequir":1,"app":[1,8],"appear":[0,1],"appelb":[0,1],"appelbdefaultvpcfilt":1,"appelbhealthcheckprotocolmismatchfilt":1,"appelblistenerfilt":1,"appelbmetr":1,"appelbtargetgroup":1,"appelbtargetgroupdefaultvpcfilt":1,"appelbtargetgroupfilt":1,"append":0,"appid":1,"appli":[0,1,7],"applic":[0,1,2,3,4,5,8],"applicationnam":1,"approach":[7,8],"appropri":0,"approxim":0,"ar":[0,1,2,4,5,7,8],"arbitrari":0,"architectur":[5,6],"aren":[0,1,7],"arg":[0,2,7],"arg_name_arg":7,"arg_type_valu":0,"argtyp":[],"argument":[0,7],"argv":0,"arithmet":2,"arithmetic_oper":0,"aritmet":[],"arm64":6,"arn":[0,1],"arn_split":[0,1],"around":[0,1],"arrai":[0,1],"arriv":[1,7],"asg":[0,1],"asid":0,"assert":[0,1,7],"assign":[0,1],"assoc":1,"associ":[0,1,2,7],"associatepublicipaddress":1,"assum":[0,1,8],"assumerolepolicysizequota":1,"assumpt":1,"assur":[0,4],"ast":[0,2,7,8],"ast2":0,"asv":1,"asvredact":1,"atm":1,"atom":[0,7],"atot":2,"attach":[0,1],"attachedinstancefilt":1,"attachedpoliciespergroupquota":1,"attachedpoliciesperrolequota":1,"attachedpoliciesperuserquota":1,"attachedvolum":1,"attachmentcount":1,"attack":1,"attempt":[0,1],"attr":1,"attribit":1,"attribut":[0,2,7],"attributesfilt":1,"atyp":7,"audit":1,"augment":1,"aurora":1,"auth":7,"author":7,"auto":1,"automat":1,"autosc":1,"autoscal":1,"avail":[0,2],"aval":1,"averag":[0,1],"avoid":[0,1,2],"aw":[0,1],"awar":2,"aws_ebs_degraded_ebs_volume_perform":1,"aws_ebs_volume_lost":1,"awsec2":1,"awslambda":1,"ax":2,"az1":1,"azr":1,"azur":1,"b":[0,2,7],"ba":1,"back":[0,1,4],"backoff":1,"backup":1,"bad1":7,"bad2":7,"balanc":[1,7],"base":[0,1,2,4,5,7,8],"base_activ":0,"base_funct":0,"based_on":0,"baseexcept":0,"baselin":1,"bash":2,"basi":1,"basic":[1,4],"batch":1,"baz":0,"bc":2,"beast":0,"becaus":[0,1,2,5,7,8],"becom":[0,1,2,4,8],"been":[0,1,8],"befor":[1,2],"behav":[0,7],"behavior":[0,1,2],"being":[0,1],"below":0,"bench":4,"benchmark":4,"better":[0,1,7],"between":[0,1,2,7,8],"beyond":[],"big":[],"bin":4,"binari":[0,4,7],"bind":[0,2,5,8],"bind_vari":0,"bit":[1,2,5,8],"blacklist":1,"blob":[0,1,4,7],"block":[0,2],"blockdevicemap":1,"bn":2,"bool":[0,5,7,8],"bool_eq":0,"bool_g":0,"bool_gt":0,"bool_l":0,"bool_lit":0,"bool_lt":0,"bool_n":0,"bool_valu":0,"boolean":[0,2,5,7],"boolean_to_statu":0,"booleangroupfilt":7,"booltyp":[0,7,8],"boolvalu":0,"both":[0,1],"boto3":1,"bottl":0,"bottom":[0,8],"bound":[0,2,7],"bracket":1,"branch":4,"break":0,"breakdown":7,"bring":1,"broken":[1,4],"broker":1,"bucket":0,"bucketencrypt":1,"bucketnotificationfilt":1,"buffer":0,"build":[0,1,4,7,8],"build_macro_ev":0,"build_reduce_macro_ev":0,"build_ss_macro_ev":0,"buildsecuritygroupfilt":1,"buildsubnetfilt":1,"buildvpcfilt":1,"built":[0,1,2,4,7,8],"bulki":1,"bunch":1,"burden":2,"bye":2,"byol":1,"byt":0,"byte":[0,8],"bytes_lit":0,"bytes_valu":0,"bytestyp":[0,8],"bytesvalu":0,"c":[0,2,7],"c4":8,"c7n":[5,8],"c7n_interfac":4,"c7n_interpreted_runn":[0,1],"c7n_s3_policy_requir":1,"c7ncontext":[0,1],"c7nlib":[1,3,5,7,8],"cach":[0,5,8],"calcul":2,"call":[0,1],"callabl":[0,8],"can":[0,1,2,3,4,5,6,7,8],"cannot":[0,1],"capabl":[0,1,2],"capacitydelta":1,"capitalon":1,"capon":1,"captur":0,"cardda_tagcompli":1,"cardin":1,"case":[0,1,2,4,6,7,8],"categori":[1,7],"caught":0,"caus":0,"causesresourc":1,"cbhfqv":1,"cbr":1,"cde":1,"ce":7,"cel":[0,1,4,6,7],"cel_activ":[0,1],"cel_ast":[0,1],"cel_env":[0,1],"cel_ev":0,"cel_expr":0,"cel_funct":7,"cel_gen":0,"cel_object":0,"cel_pars":0,"cel_prgm":0,"cel_program":7,"cel_repl":0,"cel_simple_testdata":4,"cel_sourc":7,"cel_test":7,"cel_trac":[0,2,3],"celbyt":0,"celevalerror":[0,7,8],"celfilt":[0,5,7],"celfilter_inst":0,"celfunct":[0,8],"celjsondecod":0,"celjsonencod":0,"celpars":[5,8],"celparseerror":0,"celpi":[1,2,3,5,7,8],"celstr":0,"celsyntaxerror":[0,7],"celtyp":[1,2,3,5,7,8],"celunsupportederror":0,"central":[0,7],"cert":1,"certain":[0,1],"certif":1,"cet":0,"cf":1,"cfn":1,"chain":[0,8],"chainmap":[0,8],"challeng":4,"chang":[0,1,4,7],"char":7,"charact":2,"character":0,"check":[0,4,7],"check_circular":0,"check_custom_origin":1,"checked_except":0,"checker":7,"child":0,"childnr":0,"children":0,"choci":[],"choic":[0,8],"choos":0,"chosen":0,"chunk":1,"ci":1,"cidr":0,"cidr_contain":0,"cidr_siz":[0,1],"cidrblock":1,"cipher":1,"circuit":[0,7],"cl":0,"claim":7,"class":[0,1,3,7,8],"classic":1,"classmethod":0,"claus":[1,4,7],"clean":[1,4],"clear":[1,2],"cli":[0,5,8],"client":[0,1],"clock":1,"clone":0,"close":[0,1,2,7],"closur":0,"cloud":[0,1,5,8],"cloudform":1,"cloudfront":[0,1],"cloudhsm":1,"cloudsearch":1,"cloudtrailen":1,"cloudwatch":[0,1],"cluster":1,"cm":1,"cm6aws11":1,"cmdbenviron":1,"cmdloop":0,"cml":1,"co":7,"code":[0,1,2,4,7,8],"codebuild":1,"codebuildproject":1,"codecommit":1,"codepipelin":1,"coerc":0,"coercion":[0,7],"collab":[],"collabor":0,"collect":[0,1,4,8],"collis":[0,1],"color":1,"column":[0,1],"com":[0,1,2,4,7],"combin":[0,1,5,7],"come":[0,1,2],"command":[0,2,6,8],"comment":[1,4],"common":[0,4,7,8],"commonli":0,"commun":1,"commut":0,"compar":[0,1,2],"comparablevers":[0,1],"comparison":[0,1,2,4,7],"compat":0,"compil":[0,1,4,6,7],"compiledrunn":[0,8],"complement":1,"complet":[0,7],"completekei":0,"complex":[0,1,2,5,7,8],"complex_express":4,"compli":1,"compliant":[1,7],"complic":[0,7],"compon":[0,1,3,5],"compons":[],"compos":[1,7],"composit":1,"compromis":7,"comput":[0,1,2,7,8],"computeenviron":1,"computesgfilt":1,"computesubnetfilt":1,"conceal":1,"concept":[0,7],"conceptu":8,"concern":[0,2],"conclud":4,"concret":0,"condit":[0,1,8],"conditionaland":0,"conditionalor":0,"configen":1,"configrul":1,"configur":[1,5,6],"conflat":[],"conform":4,"confus":[0,4],"connect":1,"connectionsecuritygroupfilt":1,"connectionsubnetfilt":1,"consequ":8,"consid":[0,1,8],"consider":1,"consist":0,"consol":[3,4],"consolid":[],"constant":0,"constraint":8,"construct":[0,5,7],"constructor":0,"contain":[0,1,3,5,7],"containerdefinit":1,"content":0,"context":[2,5,7],"continu":7,"control":1,"convers":[0,2,4,7,8],"convert":[0,1,4,7],"copi":[0,1,4],"core":[0,1],"correct":1,"correctli":0,"cost":8,"could":[0,2,7],"count":[0,7],"countnumb":1,"cours":0,"cover":0,"coverag":4,"cpp":0,"cpu":1,"cpuutil":[0,1],"crash":[4,7],"creat":[0,1,4,7,8],"created":1,"createdtim":1,"creation":[0,1],"creationd":1,"creationtimestamp":7,"credenti":0,"credentialreport":[0,1],"credentialreportmixin":[0,1],"crossaccountaccess":1,"crossaccountaccessfilt":[0,1],"crossaccountaccessmixin":[0,1],"crossaccountfilt":1,"crossaccountp":1,"crossregiontransf":1,"crypto":1,"csv":1,"csv2dict":1,"ct":1,"ctl":1,"current":[0,1,2,4,5,7,8],"custodian":[1,5,8],"custodian_asv":1,"custodian_cleanup":1,"custodian_downtim":1,"custodian_invalid":1,"custodian_invalid_asg":1,"custodian_rds_offhours_ct":1,"custodian_rds_offhours_et":1,"custodian_rds_offhours_pt":1,"custodian_res":1,"custodian_s3_ns_templ":1,"custodian_snapshot":1,"custodian_statu":1,"custodian_stop":1,"custodian_tag":1,"custom":1,"cut":0,"cw":1,"d":[0,1,2,3,7],"dai":[0,1],"daili":1,"darwin":6,"data":[0,2,4,5,7,8],"databas":[0,1],"databaseconnect":1,"dataev":1,"datapipelin":1,"datapoint":1,"date":[0,1,4],"datetim":[0,1,7,8],"dateutil":0,"dax":1,"daxsecuritygroupfilt":1,"daxsubnetfilt":1,"daylight":1,"dbclusteridentifi":1,"dbsubnetgroup":1,"dbsubnetgroupnam":1,"dd":1,"deal":0,"debug":[0,3,4],"decid":7,"decim":[0,2],"decis":1,"decl":[0,1,7],"declar":[0,1,7],"decod":0,"decompos":0,"decomposit":0,"decor":[0,3,7],"deduc":0,"deep":0,"deepli":0,"def":[0,1,7],"default":[0,2,4,7,8],"default_tz":1,"default_vpc":1,"defaultvpc":1,"defer":0,"defien":[],"defin":[0,1,2,4,8],"definit":[0,2,4,7,8],"deleg":0,"delet":[1,4,7,8],"deleteprotect":7,"delimit":[1,2],"deliv":1,"demand":1,"deni":1,"denial":1,"denot":8,"depend":[0,1,2,4,5,6,7,8],"deprec":[1,8],"deriv":0,"descend":0,"describ":[0,1,7,8],"describe_db_snapshot_attribut":[0,1],"describe_subscription_filt":[0,1],"descript":[0,1,5,7],"design":[0,5],"desir":1,"desk":2,"destin":1,"detail":[1,2,3,7,8],"detect":1,"determin":[1,2,7],"dev":[1,4,7],"develop":[0,5],"dfunction":0,"dhcpoptionsfilt":1,"dhe":1,"dict":[0,1,8],"dict_find_nam":0,"dictionari":[0,1],"did":0,"didn":7,"differ":[0,1,2,5,7],"difficult":[0,1],"digest":1,"digit":0,"dimens":[0,1],"direct":0,"directconnect":1,"directli":[0,1,2],"directori":[1,2,3,4],"directorysecuritygroupfilt":1,"directorysubnetfilt":1,"directoryvpcfilt":1,"disabl":[0,1,7],"disableapitermin":1,"discard":0,"discern":8,"discov":0,"dispatch":[],"displai":0,"displaynam":1,"distinct":[0,7,8],"distinguish":[0,7],"distribut":[1,4],"disutil":0,"diverg":1,"divid":0,"dividebyzeroerror":7,"divis":0,"dlm":1,"dm":1,"dn":1,"dnshostnam":1,"dnsname":1,"dnssupport":1,"do":[0,1,2,3,7,8],"do_by":0,"do_eof":0,"do_exit":0,"do_quit":0,"do_set":0,"do_show":0,"doc":[0,1,5],"docker":4,"dockerfil":4,"document":[0,1,2,4,7,8],"doe":[0,1,2,7,8],"doesn":[0,1,4,7,8],"doin":0,"domain":[0,1,8],"don":[0,1,2,7],"done":[0,1,3,8],"dot":5,"dot_id":[0,8],"dot_ident_arg":[0,8],"doubl":2,"double_valu":0,"doubletyp":[0,2,8],"doublevalu":0,"down":[0,1],"download":4,"dpb":0,"dramat":[],"dsl":[1,2,5,7,8],"due":1,"dump":0,"dumpast":0,"duplic":0,"durat":[0,1,7],"durationtyp":[0,1,8],"dure":[0,1,2,6],"dyn":0,"dynam":4,"dynamodb":1,"dynamodbacceler":1,"e":[0,1,2],"e1":0,"e2":0,"each":[0,1,2,7,8],"easi":0,"easier":[1,7],"easili":[0,5],"east":[0,1],"east1":7,"eastern":1,"eb":0,"ebnf":4,"ebsoptim":1,"ec":1,"ec2":[0,1,7],"ec2_userdata_stop":1,"ecdh":1,"ecdsa":1,"echo":2,"ecr":1,"ecrcrossaccountaccessfilt":1,"ecsclust":1,"ecsmetr":1,"ef":1,"effect":[0,1,2],"effort":4,"egress_viol":1,"eip":1,"either":[0,1,4],"ek":1,"ekssgfilt":1,"ekssubnetfilt":1,"eksvpcfilt":1,"elast":1,"elasticach":1,"elasticachesnapshot":1,"elasticachesnapshotag":1,"elasticbeanstalk":1,"elasticfilesystem":0,"elasticfilesystemmounttarget":1,"elasticmapreduc":1,"elasticsearch":1,"elasticsearchdomain":1,"elb":[0,1],"elblog":1,"elbsecuritypolici":1,"elem":0,"element":1,"elif":0,"elimin":[],"els":0,"email":[1,7],"email_verifi":7,"emb":2,"embed":[5,7],"emit":[],"emmo":1,"emphat":7,"empti":[0,1,3,4],"emr":1,"emrclust":1,"emrmetr":1,"en":[4,7],"enabl":[0,3,5,7],"enasupport":1,"encod":0,"encount":0,"encryptionenabledfilt":1,"end":[0,1,8],"endpoint":1,"endpointcrossaccountfilt":1,"endpointsecuritygroupfilt":1,"endpointsubnetfilt":1,"endpointvpcfilt":1,"endtim":0,"engin":[0,1],"enhanc":2,"eni":1,"ensur":[1,7],"ensure_ascii":0,"ent":1,"enter":[0,2],"enterpris":1,"entir":0,"entri":8,"enum":[0,1,4],"enumtyp":0,"env":[1,7],"environ":[0,1,3,4,5,6,7,8],"environment":[],"eof":2,"ephemeralinstancefilt":1,"eq":[0,1],"eq_test":0,"equal":[0,1],"equival":0,"err":7,"erron":0,"error":[0,2,3,4,5,8],"error_text":0,"escap":0,"eschew":1,"essenc":7,"essenti":[0,1,5,8],"est":1,"establish":0,"et":1,"etc":[0,1],"eu":1,"europ":0,"eustac":0,"eval":[0,2,7,8],"eval_error":0,"eval_filt":1,"evalerror":[],"evalu":[1,2,3,4,5,7],"evauat":[],"even":[0,1],"event":[0,4],"eventrul":1,"eventrulemetr":1,"eventruletarget":1,"eventstatuscod":0,"eventu":[0,7],"ever":7,"everi":[0,1],"everydai":1,"everyone_onli":1,"everyth":2,"evluat":[],"ex":[0,7],"ex_":0,"ex_10":0,"ex_10_l":0,"ex_10_x":0,"exact":1,"exactli":7,"examin":[0,1,2,7],"exampl":[0,5,8],"exc_class":0,"exc_typ":0,"exc_valu":0,"exceed":1,"except":[0,1,2,4,5,8],"exceptionb":[],"exceptionmanualsnapshot":1,"exclud":1,"exclude_imag":1,"exclus":[],"exec":[0,8],"execut":[0,1,2,4,7,8],"exempt":[0,1],"exist":[0,1,3],"exists_on":0,"exit":5,"expand":0,"expans":0,"expcept":[],"expect":[0,1,2,7],"experiment":8,"expir":1,"explicit":[0,1,8],"explicitli":[0,1],"explict":2,"explor":8,"export":0,"expos":[0,1,7,8],"expost":[],"expostur":[],"expr":[0,1,2,4,7,8],"expr_test_bc":4,"express":[0,1,2,4,6,7,8],"exprlist":0,"exprpb":7,"extant":1,"extend":[0,1,2,7,8],"extended_name_path":0,"extens":[0,1,7,8],"extent":[0,1,7],"extern":0,"extra":1,"extract":[0,1],"f":[0,1,2,7,8],"f1":2,"f2":2,"facad":[0,8],"factfind":1,"factori":0,"fail":[0,1],"failur":[0,1,2],"fairli":0,"fall":0,"fallback":0,"fals":[0,1,2,7],"far":0,"fargat":1,"fatal":[3,4],"fatalf":7,"fatalln":7,"faulttolerantsnapshot":1,"favor":0,"featur":[1,5,7,8],"fetch":1,"few":[0,1,7],"ffffffff":1,"field":[0,1,4,7,8],"fieldinit":[0,8],"fifo":2,"file":[0,1,3,5],"filesystem":2,"filter":[0,5,8],"filter_inst":7,"filter_registri":1,"filter_resource_statist":1,"filterrestintegr":1,"filterrestmethod":1,"final":[0,1,2,7,8],"find":[0,2],"find_nam":0,"findid":[],"firehos":1,"first":[0,7,8],"fit":[0,1,7],"fiter":0,"five":7,"fix":[0,1,7],"fixedint":0,"fixedtz":0,"fixtur":0,"fklter":1,"flag":[1,4],"flat":0,"flatten":[],"flavor":0,"fleet":1,"float":[0,2,8],"float_lit":0,"floatvalu":0,"flow":0,"flow_log":[0,1],"flowlogfilt":1,"fmt":7,"fnmatch":[0,1],"focus":1,"follow":[0,1,2,3,4,7,8],"foo":1,"foobar":1,"forbidden":1,"form":[0,1,4,7,8],"formal":[0,7],"format":[0,1,2,3,4,7],"formatt":3,"formerli":0,"forwardref":0,"found":[0,1,8],"foundat":0,"four":[0,1,7,8],"fp_math":4,"fraction":0,"fragment":0,"frame":0,"framework":[7,8],"fraught":4,"fri":1,"friendli":[0,2],"from":[0,1,2,4,5,8],"fromport":1,"front":0,"frozenset":0,"fs_analytical_dev":1,"fs_analytical_qa":1,"fs_core_cas_qa":1,"fs_custodian_tag":1,"fs_manual_ebs_snapshot_expir":1,"fs_manual_rds_snapshot_expir":1,"fsx":1,"fulfil":0,"full":[0,4],"full_control":1,"fulli":0,"func":[0,7],"func_nam":0,"function":[2,5,8],"function_contain":0,"function_endswith":0,"function_ev":0,"function_getd":0,"function_getdayofmonth":0,"function_getdayofweek":0,"function_getdayofyear":0,"function_getfullyear":0,"function_gethour":0,"function_getmillisecond":0,"function_getminut":0,"function_getmonth":0,"function_getsecond":0,"function_match":0,"function_s":0,"function_startswith":0,"functiontyp":[0,7],"functool":7,"fundament":8,"further":0,"futur":0,"g":[0,1,2,4],"g4":0,"gamelift":1,"gatewai":1,"gather":[1,4,7],"gb":1,"gc":1,"gcm":1,"ge":[0,1],"gen":0,"gener":[0,1,4,7,8],"geograph":1,"get":[0,1,2,4,8],"get_access_log":[0,1],"get_account":[0,1],"get_credential_report":[0,1],"get_dbsubnet_group_us":[0,1],"get_endpoint":[0,1],"get_flow_log":1,"get_health_ev":[0,1],"get_instance_imag":[0,1],"get_key_polici":[0,1],"get_launch_configuration_nam":[0,1],"get_load_balanc":[0,1],"get_matching_alias":[0,1],"get_metr":[0,1],"get_metric_statist":1,"get_model":1,"get_nam":0,"get_opt":0,"get_orgid":[0,1],"get_protocol":[0,1],"get_raw_health_ev":[0,1],"get_raw_metr":[0,1],"get_rel":[0,1],"get_related_id":[0,1],"get_related_igw":0,"get_related_kms_kei":0,"get_related_nat_gatewai":0,"get_related_security_config":0,"get_related_sg":0,"get_related_subnet":0,"get_related_vpc":0,"get_resource_polici":[0,1],"get_resource_statist":[0,1],"get_resource_valu":1,"get_type_protect":[0,1],"get_valu":[0,8],"get_vpc":[0,1],"getdat":[0,1],"getdayofmonth":0,"getdayofweek":[0,1],"getdayofyear":0,"getfullyear":0,"gethour":[0,1],"getmillisecond":0,"getminut":0,"getmonth":0,"getsecond":0,"gettz":0,"gherkin":4,"gib":1,"git":4,"github":[0,1,2,4,7],"give":7,"given":[0,1,2,4,7,8],"gl":1,"glacier":1,"glaciercrossaccountaccessfilt":1,"glob":[0,1],"global":0,"globalgrantsfilt":1,"glue":1,"glueconnect":1,"go":[0,1,4,5],"go_mod":0,"goal":0,"golang":[0,4],"goo":1,"good":[1,7],"googl":[0,1,2,4,7],"gracefulli":0,"grai":1,"grammar":[0,4],"grantcount":1,"great":[],"greater":1,"greet":7,"greetfunc":7,"group":[0,7],"group_access":2,"groupid":1,"groupmembership":1,"groupnam":1,"grouppolicysizequota":1,"groupset":1,"groupsperuserquota":1,"groupsquota":1,"gt":[0,1],"gte":1,"h":[0,1,2],"ha":[0,2,4,5,7,8],"had":1,"hand":7,"handl":[0,1,2,7,8],"handler":3,"hapg":1,"happen":0,"hard":0,"hash":0,"hashabl":0,"hasn":8,"hasstatementfilt":1,"hasvirtualmfa":1,"have":[0,1,2,3,4,5,7,8],"haven":1,"head":[0,1,4],"header":1,"healtch":1,"health":0,"health_ev":1,"healthcheckprotocolmismatch":1,"healthev":1,"healtheventfilt":1,"healthfilt":1,"heavi":1,"hello":[0,7],"help":[0,1,2,4,7,8],"here":[0,1,2,7,8],"heredoc":2,"hexadecim":[],"hh":0,"hidden":0,"higher":1,"highli":1,"hint":[0,4,8],"hit":1,"holidai":1,"home":[1,2,3],"hook":0,"host":[0,1],"hostedzon":1,"hostnam":1,"hour":[0,1],"hourli":1,"housekeep_unused_sg":1,"how":[0,1,7],"howev":[0,1,7,8],"hsm":1,"html":[0,1,4,7],"http":[0,1,2,4,5,7],"hybrid":0,"i":[0,2,4,5,6,7,8],"iam":0,"iamaccessfilt":[0,1],"iamgroupinlinepolici":1,"iamgroupus":1,"iaminstanceprofil":[0,1],"iamroleinlinepolici":1,"iamroleusag":[0,1],"iamroleusagemixin":[0,1],"iamsummari":1,"iamuserinlinepolici":1,"iana":[0,1],"id":0,"idea":[0,7],"ideal":0,"ident":[0,1,8],"ident_arg":[0,8],"ident_pat":0,"ident_valu":0,"identif":[],"identifi":[0,1,8],"ie":1,"ietf":0,"ifi":2,"ignor":[0,1,2,7],"ii":0,"imag":[0,4],"imageag":1,"imageagefilt":1,"imagefilt":1,"imageid":[0,1],"imagenam":1,"imagesunusedmixin":[0,1],"imageunusedfilt":[0,1],"immedi":1,"implement":[0,2,4,5,8],"implent":1,"impli":[0,1],"implicit":1,"implicitli":1,"import":[0,1,2,4,5,7,8],"improt":8,"in_tre":[],"includ":[0,1,3,4,5,7,8],"incomplet":0,"incorrect":1,"incorrectlambda":1,"increas":0,"inde":[],"indent":0,"independ":0,"index":[0,4,5],"indexerror":0,"indic":[0,2],"individu":[0,1],"infer":1,"infin":0,"info":[1,3],"inform":[0,1,4,7,8],"infrastructur":5,"init":0,"initi":0,"inject":0,"input":[0,2,7,8],"insid":[0,8],"instal":[4,5],"instanc":[0,7,8],"instance_profile_usag":[0,1],"instanceagefilt":1,"instanceattribut":1,"instancecreatetim":1,"instanceid":0,"instanceimag":1,"instanceimagebas":[0,1],"instanceimagemixin":[0,1],"instanceinitiatedshutdownbehavior":1,"instanceoffhour":1,"instanceonhour":1,"instanceprofil":1,"instanceprofilenam":1,"instanceprofilesquota":1,"instances":7,"instancestorag":1,"instancetyp":1,"instead":[0,1,2,7],"insufficient_data":1,"int":[0,1,2,8],"int32":0,"int32valu":0,"int64":0,"int64_overflow_neg":0,"int64_overflow_posit":0,"int64_valu":[0,4],"int64valu":0,"int_lit":0,"intact":[0,4],"integ":[0,1,2],"integer_math":[0,4],"integr":[0,2,3,8],"intend":0,"intent":[1,2,5],"interact":[0,2,8],"interect":0,"interest":[5,8],"interfac":[0,1,7,8],"interfacesecuritygroupfilt":1,"interfacesubnetfilt":1,"interfacevpcfilt":1,"interim":4,"intermedi":[0,7],"intern":[0,1,7,8],"internet":1,"interoper":5,"interpret":0,"interpretedrunn":[0,8],"interrog":0,"intersect":[0,1],"intiial":[],"intoper":0,"intro":0,"introduc":[0,7,8],"introspect":[],"inttyp":[0,2,4,8],"invalid":2,"invalidconfigfilt":1,"invers":1,"invis":1,"involv":[0,2,5,7,8],"io":[0,7],"iop":1,"iot":1,"ip":1,"ipaddress":0,"ippermiss":1,"ippermissionegress":1,"ipv4address":0,"ipv4network":0,"is_log":[],"ish":4,"islog":1,"isloggingfilt":[0,1],"isn":[0,1,8],"isnotloggingfilt":[0,1],"isqueryloggingen":1,"isrm":1,"iss":7,"isshadow":1,"isshieldprotect":[0,1],"isshieldprotectedmixin":[0,1],"issslfilt":1,"issu":[0,1,7,8],"iswafen":[0,1],"item":[0,1,2,7],"iter":[0,1],"its":[0,1,7],"itself":0,"iunjpz":1,"jme":0,"jmes_path":[0,1],"jmes_path_map":[0,1],"jmespath":[0,1],"job":[0,1],"joda":0,"jq":[0,2,8],"json":[0,1,2,4,7,8],"json_queri":4,"json_to_cel":[0,1,7],"jsonl":0,"jsonlin":2,"just":1,"kai":0,"keep":[0,1],"kei":[0,5,7,8],"kernel":1,"key_id":0,"key_valu":1,"keyalia":1,"keyerror":0,"keyid":1,"keynam":1,"keyrotationen":1,"keyrotationstatu":1,"kind":[0,1,2,7,8],"kinesi":1,"kit":4,"km":0,"kms_alia":[0,1],"kms_kei":[0,1],"kmsfilter":1,"kmskeyalia":1,"known":[0,2,7],"kwarg":0,"l":2,"laast":[],"label":0,"lambda":[0,1],"lambdacrossaccountaccessfilt":1,"lambdaeventsourc":1,"lambdalayervers":1,"langdef":[0,1,4],"languag":[0,1,2,4,7,8],"larg":1,"large_instance_typ":1,"large_resource_set":4,"lark":[0,4,7,8],"last":0,"last_rot":1,"last_used_d":1,"last_used_region":1,"last_used_servic":1,"lastrot":1,"lastupdatedtim":1,"lastwritedai":1,"later":[0,1],"latest":[0,7],"latestsnapshot":1,"launchconfig":1,"launchconfigag":1,"launchconfigfilt":1,"launchconfigurationnam":1,"launchtyp":1,"layer":[0,1,8],"layercrossaccount":1,"lc_userdata":1,"le":[0,1],"lead":[0,1],"learn":4,"least":[0,1],"leav":1,"left":[0,1,4],"legaci":[0,1],"len":[0,1],"length":0,"less":[0,1,7],"let":0,"letter":2,"level":[0,1,3,4,8],"levelnam":3,"leverag":[0,8],"lexic":0,"lh":7,"lib":4,"librari":[0,1,4,7,8],"licens":1,"life":[0,1],"lifecyclerul":1,"like":[0,1,2,3,4,7],"likewis":1,"limit":0,"line":[0,1,2,8],"lineno":3,"link":2,"linkag":1,"lint":4,"linux":2,"list":[0,1,4,7,8],"list_lit":[0,8],"list_typ":[],"listtyp":[0,8],"listvalu":0,"liter":[0,1,2,8],"littl":[0,2,4],"ll":[0,7,8],"load":[0,1],"load_annot":0,"load_valu":0,"local":[0,1,3,4,8],"localized_ev":0,"locat":0,"log":[0,2,3,7],"logcrossaccountfilt":1,"logg":1,"logger":[0,3],"loggroup":1,"logic":[0,2,4,7],"logical_and":0,"logical_condit":0,"logical_not":0,"logical_or":0,"logtarget":1,"long":[0,1],"longer":[1,2],"longest":0,"longtz":0,"look":[0,1,4,7],"lookup":0,"loos":[],"loosevers":0,"los_angel":1,"loss":1,"lot":[],"lower":[0,1],"lowercas":0,"lt":[0,1],"lte":1,"m":[0,1,2,6,7],"m1":7,"m3":1,"machin":1,"machineri":[],"macro":[0,1,4,8],"macro_":0,"macro_al":0,"macro_exist":0,"macro_exists_on":0,"macro_filt":0,"macro_has_ev":0,"macro_map":0,"made":[0,1],"magnet":1,"magnitud":0,"mai":[0,1,4,6,7,8],"maid":1,"maid_offhour":[0,1],"maid_statu":1,"main":[0,8],"major":1,"make":[0,1,2,7,8],"makefil":5,"manag":0,"mani":[0,1,2],"manual":[1,4],"map":[0,1,2,7,8],"map_lit":[0,8],"map_typ":[],"mapinit":0,"mapkeytyp":8,"mappubliciponlaunch":1,"maptyp":[0,1,7,8],"march":0,"mark":[],"marked_kei":[0,1],"markedforop":1,"marker":[0,4],"mask":[0,7],"master":[0,1,4,7],"match":[0,1,4,6,7,8],"matchabl":1,"math":[0,2],"matter":0,"max":1,"maximum":1,"maxproperti":1,"maxsecond":0,"mayb":0,"mccarthi":0,"md":[0,1,4,7],"me":7,"mean":[0,1,2,7,8],"meaning":0,"medium":1,"meet":7,"member":[0,1,5],"member_dot":[0,8],"member_dot_arg":[0,8],"member_index":0,"member_item":[0,8],"member_object":[0,8],"mental":2,"mere":0,"messag":[0,1,3,4,7],"message_liter":0,"messagebrok":1,"messagetyp":[0,8],"meta":0,"method":[0,3,4,7,8],"method_ev":0,"method_id":0,"metric":0,"metricnam":0,"metricsaccess":0,"metricsfilt":1,"mfa_act":1,"mfadevic":1,"mfadevicesinus":1,"mi":1,"middl":0,"might":[0,7],"mimic":2,"min":1,"mini":2,"minim":[0,3,5],"minimum":[0,1],"minimumpasswordlength":1,"minproperti":1,"minsecond":0,"minut":[0,1],"mismatch":0,"mismatchs3origin":1,"miss":[0,5],"missingrout":1,"mistak":1,"mix":0,"mixin":[0,1],"mkgherkin":4,"ml":1,"mlstring_lit":0,"mm":[0,1],"mock":[0,1],"mod":[0,1],"mode":[0,2],"model":[0,1,7],"modern":1,"modif":[1,2],"modifi":[0,1],"modifyablevolum":1,"modul":[0,1,3,4,5,7,8],"modulo":0,"modulu":0,"moment":1,"mon":1,"monad":0,"monitor":1,"monthli":1,"more":[0,1,2,3,4,5,8],"morn":1,"most":[0,1,2,4,7,8],"mount":1,"mq":1,"mqmetric":1,"mqsgfilter":1,"mqsubnetfilt":1,"mssage":1,"much":[0,2],"mul":0,"multi":1,"multilin":2,"multipl":[0,1,2,7,8],"multipli":1,"multiplication_div":0,"multiplication_mod":0,"multiplication_mul":0,"must":[0,1,2,3,4,7,8],"my":1,"my_extens":7,"my_extns":[],"mydata":1,"n":[0,2,7],"nacl":1,"name":[1,2,3,4,5,7],"name1":0,"name2":0,"name_arg":7,"name_token":0,"namecontain":[0,3,8],"nameerror":[0,8],"namespac":[0,1,4],"nano":0,"nanosecondspersecond":0,"narrowli":1,"nat":1,"nativ":[0,1,7],"navgiat":[],"navig":0,"nc":[0,1],"nc1":0,"nc2":0,"ndjson":[0,2],"ne":[0,1],"necessari":0,"necessarili":0,"need":[0,1,3,5,7,8],"needless":0,"needlessli":[],"neg":0,"neither":[4,8],"nest":[0,7,8],"nested_activ":0,"net":[0,1],"networkacl":1,"networkin":1,"networkinterfac":1,"networkloc":1,"networkout":1,"new":[0,1,7],"new_activ":0,"new_text":0,"newenv":7,"newer":7,"newfunct":7,"newinstanceoverload":7,"newli":1,"newlin":2,"newoverload":[],"newvar":7,"next":[0,1,8],"ni":1,"nice":[0,7],"nightli":1,"nil":7,"nljson":[0,2],"no_such_field":0,"node":0,"nodesnaphot":1,"nomin":0,"non":[0,3,5,7],"non_compli":1,"none":[0,1,7],"nonetyp":0,"nonpubl":1,"nor":8,"noreturn":0,"normal":[0,1,2],"nospecificiamrolemanagedpolici":1,"not_":0,"not_applic":1,"not_nul":1,"notabl":2,"notact":1,"note":[0,1,2,7,8],"notebook":1,"notebookinst":1,"notebooksecuritygroupfilt":1,"notebooksubnetfilt":1,"notencryptedfilt":1,"notfound":0,"noth":[0,1],"notifi":1,"notimplementederror":7,"notion":0,"notprincip":1,"notresourc":1,"notsetsentinel":0,"nov":1,"now":[0,1,7],"nsib":[],"nsj7vg":1,"ntp":1,"nuanc":[0,8],"null":[0,1,2],"null_lit":0,"null_typ":[],"null_valu":0,"nulltyp":[0,8],"nullvalu":0,"number":[0,1,2,4,7,8],"number_valu":0,"numer":7,"o":[0,1,4],"obect":0,"object":[2,5,7,8],"object_hook":0,"object_pairs_hook":0,"obscur":[0,1],"obsolet":0,"obvers":1,"obviou":7,"occur":[1,7],"octal":0,"odd":0,"off":[0,1],"offer":[0,1,8],"offhour":0,"offhours_except":1,"offset":[0,1],"often":[0,4,5,7],"ok":1,"old":[0,1],"older":1,"omit":[1,2,7],"onc":[0,1],"one":[0,1,2,4,5,7,8],"oneof":[0,1],"onli":[0,1,2,3,4,7],"onlin":1,"onlyport":1,"onto":0,"oop":[0,7],"op":[0,7],"op_name_map":0,"opa":1,"open":[0,1],"openpolicyag":[0,1],"opensourc":5,"oper":[0,1,2,7,8],"operand":[0,1],"operator_in":0,"opswork":1,"opt":1,"optim":[0,1,7],"option":[0,2,3,6],"order":[0,1],"ordinari":0,"oreo":1,"org":[0,1,2,4],"organ":[1,5],"organi":8,"organiz":1,"orgid":0,"origin":[0,7],"other":[0,1,2,3,4,6,7,8],"otherwis":0,"out":[0,1,4,7],"outlin":[1,7],"output":[2,3,4,7],"outsid":[0,1,7,8],"over":[0,1,2,4,7,8],"overal":[0,7,8],"overdraftlimit":7,"overdraftprotect":7,"overflow":0,"overhead":8,"overlap":0,"overload":[0,7,8],"overloadd":0,"overrid":[0,1,8],"overridden":0,"overwhelm":1,"overwrit":4,"own":[0,1,7],"owner":[1,7],"ownercontact":1,"ownereid":1,"p":[0,2],"paackag":[],"pacif":1,"packag":[0,2,3,5,6,7,8],"packagetyp":0,"page":5,"pair":[0,1],"paragraph":3,"parallel":[0,2,4],"param":[],"paramet":[0,3,7],"parameterfilt":1,"paren_expr":[0,8],"parent":[0,1],"parent_it":0,"pari":0,"pars":[0,1,4,8],"parse_cidr":[0,1],"parse_const":0,"parse_d":1,"parse_float":0,"parse_int":0,"parse_text":[0,1],"parser":[0,1,2,7,8],"part":[0,1,4,5,7,8],"partial":[0,7],"particular":[0,1,2],"particularli":1,"partit":0,"partivular":0,"pass":[0,1,4,7],"password_en":1,"password_last_chang":1,"password_last_us":1,"password_next_rot":1,"passwordpolicyconfigur":1,"past":1,"patch":1,"path":[0,1,4],"path_sourc":0,"pattern":[0,1,2],"payload":0,"pb2g":5,"peer":1,"peeringconnect":1,"pendulum":[0,1],"per":1,"percent":1,"perfectli":1,"perform":[1,4,8],"perhap":0,"period":[0,1],"permiss":[1,2],"permissionsboundaryusagecount":1,"permit":[0,1,3,7,8],"persist":0,"person":8,"perspect":7,"pg":1,"pgm":[0,1],"phase":0,"phase1transpil":0,"phase2transpil":0,"phd":1,"phoni":4,"pick":2,"piec":[1,2],"pingstatu":1,"pip":[4,6],"pipe":2,"place":[0,1,2],"plan":[0,1],"platform":[1,4,5,6],"platformnam":1,"platformvers":1,"pleas":1,"pleasant":0,"plu":[0,1],"pluck":0,"plumb":4,"pm":1,"poetri":6,"point":[0,1,2,7,8],"polici":[0,5,7],"policiesquota":1,"policyassign":1,"policynam":1,"policysizequota":1,"policyversionsinus":1,"policyversionsinusequota":1,"polit":0,"pollut":3,"pool":[0,1],"poorli":0,"pop":0,"popul":0,"popular":1,"port":1,"posit":7,"possibl":[0,1,2,8],"possibli":[0,1],"post":1,"power":2,"pprint":0,"pragmat":[0,7],"prcess":1,"precis":4,"predefin":8,"prefer":0,"prefetch_instance_imag":0,"prefix":[0,1],"prefixlen":[0,1],"preloop":0,"prepar":[1,8],"prepare_queri":[0,1],"presenc":[0,1,7],"present":[0,1,4],"preserv":1,"presum":2,"pretti":0,"prevent":[0,1,7],"previou":[0,1,7],"previous":1,"prg":7,"prgm":[0,7],"primari":[0,2,8],"primarili":0,"primit":[1,2],"princip":1,"principl":[0,5],"print":0,"println":7,"prior":[0,7],"probabl":1,"problem":[0,1,4,8],"process":[0,1,2,4,7],"process_json_doc":0,"process_resource_set":1,"process_value_typ":1,"prod":7,"prodlog":1,"produc":[0,4,7],"product":[0,5],"productcod":1,"profil":1,"program":[0,1,2,5,7,8],"programopt":7,"progress":0,"project":[0,1,5,7],"promot":1,"prompt":[0,2],"propag":[0,1],"proper":[0,1],"properli":6,"properti":[0,1,7],"propog":0,"propos":1,"proto":[0,7],"proto2":4,"proto3":4,"proto3pb":0,"protobuf":[0,1,2,4,7],"protobuf_messag":8,"protobuf_typ":0,"protobyf":0,"protocol":[0,5],"protubuf":[],"provid":[1,2,3,4,7,8],"provis":[0,1],"proxi":1,"pst":1,"pt":[0,1],"pubfac":1,"public":0,"pull":[1,4],"purchas":1,"pure":0,"purpos":1,"push":0,"put":1,"py":[0,1,4],"pycodeblock":[],"pypi":0,"python":[0,1,4,6,7,8],"pythonpath":[],"pythontranspil":0,"qa":1,"qualifi":0,"qualified_identifier_resolution_uncheck":0,"queri":0,"question":0,"queue":1,"quick":4,"quickli":5,"quietli":0,"quit":[0,2,8],"quot":2,"quota":1,"quux":0,"r":[0,2,7],"r53domain":1,"radisti":1,"raft":1,"rais":[0,2,3,7,8],"raisw":[],"ramdisk":1,"rang":[0,8],"rank":1,"rare":[0,1],"raw":[0,1],"rd":[0,1],"rdscluster":1,"rdsclustersnapshot":1,"rdsoffhour":1,"rdsonhour":1,"rdssnapshot":1,"rdssnapshotag":1,"rdssnapshotonhour":1,"rdssubnetgroup":1,"rdt":1,"re":[0,1,4],"re2":6,"reach":[0,1],"read":[0,1,2,4,7],"read_acp":1,"readi":[0,7],"readm":[0,7],"readonli":1,"readonlyrootfilesystem":1,"readreplicadbinstanceidentifi":1,"readreplicasourcedbinstanceidentifi":1,"readthedoc":7,"real":0,"realli":[0,2],"reason":[1,5],"receiv":7,"recent":[0,1,4],"recogn":0,"recommend":1,"recov":1,"recurs":0,"redact":1,"redshift":1,"redshiftsnapshot":1,"redshiftsnapshotag":1,"redshiftsnapshotcrossaccount":1,"reduc":[0,1],"reduct":0,"ref":[0,1,7],"ref_to":0,"refactirubg":1,"refactor":[0,1],"refer":[0,1,2,7,8],"referenc":1,"reflect":[0,4,7],"refractor":[],"refresh_period":1,"refus":0,"regex":[0,1],"region":[0,1],"regist":[0,1],"registri":[0,1],"regular":[1,2,6],"rehydr":[0,1],"reject":1,"rel":0,"relat":[0,1,7,8],"relatedresourcefilt":[0,1],"relatedresourcemixin":[0,1],"relation_eq":0,"relation_g":0,"relation_gt":0,"relation_in":0,"relation_l":0,"relation_lt":0,"relation_n":0,"relationshiup":1,"releas":1,"reli":[0,1,2,4,7],"remain":1,"remaind":0,"remot":0,"remov":[0,1,4],"renam":[0,1,8],"reorder":1,"repeat":0,"repeatedli":1,"repl":[0,2,3],"replac":[0,1,7,8],"replicationinst":1,"report":[1,4],"report_delai":1,"report_gener":1,"report_max_ag":1,"repositori":4,"repr":0,"repres":[0,1],"represent":[0,1],"reproduc":4,"request":[0,1,7],"requie":[],"requir":[0,2,4,5,7,8],"require_ssl":1,"requiredencryptedputobject":1,"requireencryptedputobject":1,"requirelowercasecharact":1,"requirenumb":1,"requiresslaccessrdt":1,"requiresymbol":1,"requireuppercasecharact":1,"rerefer":0,"reservedconcurr":1,"resili":1,"resiz":1,"resize_config":1,"resolut":[7,8],"resolv":[0,7],"resolve_funct":0,"resolve_nam":[0,8],"resolve_vari":0,"resourc":[0,1],"resource_count":[0,1],"resource_list":7,"resource_schedul":[0,1],"resource_typ":[1,7],"resourcekmskeyalia":[0,1],"resourcekmskeyaliasmixin":[0,1],"respect":0,"rest":2,"restapi":1,"restapicrossaccount":1,"restor":[0,1],"restresourc":1,"restrict":1,"result":[0,1,2,7,8],"result_funct":0,"result_valu":0,"resum":1,"resut":[],"retriev":2,"return":[0,1,2,7,8],"reus":[0,1],"revers":[0,1],"revis":[1,4],"rework":1,"rewrit":[4,7],"rewritten":0,"rfc":0,"rfc3339":0,"rh":7,"rhymjmbbe2":1,"rich":0,"right":[0,8],"role":[0,1],"rolecrossaccountaccess":1,"rolenam":1,"rolepolicysizequota":1,"rolesquota":1,"roll":4,"root":[0,1,3],"root_scop":0,"rootdevicenam":1,"route53":1,"routet":1,"rrset":1,"rsa":1,"rst":[],"rule":[0,7],"rulestatu":1,"run":[0,1,4,5,7,8],"runnabl":[],"runner":[0,3,7,8],"runner_class":[0,1],"runtim":0,"s3":0,"s3bucketnam":1,"s3crossaccountfilt":1,"s3metric":1,"s3publicblock":1,"sagemak":1,"sai":[0,1],"said":0,"same":[0,1,2,4,7,8],"samplecount":1,"save":[0,1,2],"sc":1,"scale":[0,1],"scan":[1,4],"scan_group":[0,1],"scatter":1,"scenario":[0,4],"schedul":1,"schedulepars":[0,1],"schema":[1,7],"scope":0,"script":8,"se":1,"search":[0,1,5],"searchabl":0,"sechema":[],"second":[0,1],"secondari":1,"secondarili":0,"secret":[1,7],"secretsmanag":[0,1],"section":[1,4,8],"secur":[0,2,5],"securetransport":1,"security_group":[0,1],"security_group_id":0,"securitygroup":1,"securitygroupdifffilt":1,"securitygroupfilt":1,"securitygrouplockedfilt":1,"see":[0,1,2,3,4,7,8],"seed":0,"seem":[0,1,8],"select":[0,1],"selector":1,"selector_valu":1,"self":[0,1,4,7],"self_eval_int_zero":4,"self_eval_uint_zero":4,"self_eval_zeroish":4,"selfcheck":4,"selfrefer":1,"semant":[0,1,2,4,5,8],"semicolon":1,"send":1,"sens":8,"sensibl":[0,1],"sentinel":[0,1],"separ":[0,1,2,7,8],"seper":1,"sequenc":[0,1,7,8],"serial":[2,8],"serializ":0,"seriou":0,"serv":0,"server":[0,1],"servercertif":1,"servercertificatesquota":1,"servic":[0,8],"service_role_usag":[0,1],"servicelimit":1,"servicemetr":1,"servicetaskdefinitionfilt":1,"sessioncontext":1,"sessionissu":1,"set":[0,1,2,3,4],"set_activ":0,"set_annot":1,"setgid":2,"setuid":2,"setup":1,"sever":[0,1,2,4,8],"sg":1,"sg_unus":1,"sgdefaultvpc":1,"sgusag":[0,1],"sgusagemixin":[0,1],"sha256":1,"sha384":1,"shadow":0,"shake":7,"shake_hand":7,"shake_hands_string_str":[],"shakefunc":[],"share":1,"shell":[2,8],"shield":0,"shield_protect":[0,1],"shield_subscript":[0,1],"shielden":[0,1],"shieldenabledmixin":[0,1],"shop":1,"short":[0,7],"shorthand":[0,1],"should":[0,1,4,8],"show":[0,2,3,7],"shown":0,"shrink":1,"si":1,"sid":1,"side":[],"sign":[0,8],"signatur":7,"signifi":1,"significantli":6,"signingcertificatesperuserquota":1,"silenc":[0,1],"silent":[0,4,7],"similar":[0,1,2,7],"similarli":[0,1],"simpl":[0,4,8],"simple_test":[],"simpledb":1,"simpler":[0,1],"simplest":[],"simpli":[0,7],"simplifi":0,"sinc":[0,1,2],"singl":[0,1,2,7],"single_bool":0,"single_byt":0,"single_doubl":0,"single_dur":0,"single_fixed32":0,"single_fixed64":0,"single_float":0,"single_int32":0,"single_int64":0,"single_sfixed32":0,"single_sfixed64":0,"single_sint32":0,"single_sint64":0,"single_str":0,"single_timestamp":0,"single_uint32":0,"single_uint32_wrapp":0,"single_uint64":0,"singledispatch":7,"singleton":[0,5,7],"singletonfilt":1,"singular":0,"situat":1,"size":[0,1,7],"size_parse_cidr":[0,1],"skew":1,"skew_hour":1,"skier":1,"skip":0,"skipkei":0,"slice":[],"slight":[1,7],"slightli":[0,1,7,8],"slighyli":7,"slower":1,"slowli":4,"slurp":[0,2],"sm":1,"small":7,"smaller":0,"smarter":8,"smi":1,"sn":[0,1],"snapshot":0,"snapshotag":1,"snapshotcreatetim":1,"snapshotcrossaccountaccess":1,"snapshotid":1,"snapshotskipamisnapshot":1,"snapshottyp":1,"snapshotunusedfilt":[0,1],"snapshotunusedmixin":[0,1],"snowbal":1,"snscrossaccount":[0,1],"snscrossaccountmixin":[0,1],"so":[0,4,7],"socket":2,"some":[0,1,2,3,7,8],"some_activ":0,"some_dict":0,"some_str":0,"someparam":1,"someth":[0,1],"sometim":[0,7],"somev":1,"somewhat":[0,7,8],"sophist":[0,7],"sort":7,"sort_kei":0,"sourc":[0,4,7],"source_data":0,"source_text":0,"sourcedestcheck":1,"sourceforg":0,"span":1,"sparingli":1,"spawn":1,"spec":[0,1,2,4],"special":[0,1,4,7],"specif":[0,3,4,5,7,8],"specifi":1,"specificaiton":4,"specificiamrolemanagedpolici":1,"speed":6,"spell":1,"sphinx":4,"spin":1,"spite":7,"spread":[0,2],"sprintf":7,"sq":1,"sql":1,"sqlserveraudit":1,"sqscrossaccount":1,"squar":1,"src":4,"sriovnetsupport":1,"ssd":1,"ssh":1,"sslnegot":1,"sslpolici":1,"sslpolicyfilt":1,"sslv2":1,"sslv3":1,"ssmstatu":1,"st":1,"st_atim":2,"st_birthtim":2,"st_blksize":2,"st_block":2,"st_ctime":2,"st_dev":2,"st_flag":2,"st_gen":2,"st_ino":2,"st_mtime":2,"st_nlink":2,"st_rdev":2,"st_size":2,"stack":[0,1],"stage":1,"stand":[0,1,8],"standard":[0,1,2,7],"start":[0,1,3,4,7,8],"startswith":7,"starttim":[0,1],"stat":[0,2],"state":[0,7],"statement":[0,7,8],"statement_id":1,"statetransitionag":1,"stati":2,"static":0,"statist":[0,1],"statu":[0,5],"status":[0,1],"stdin":[0,2],"stdout":0,"stem":4,"step":[0,1,4,7],"stick":[],"sticki":2,"still":[2,4],"stop":1,"storag":1,"store":1,"stori":[],"str":[0,1,8],"stream":[0,1],"streamhandl":3,"strict":0,"strictli":0,"string":[0,1,2,4,7],"string_greet_str":7,"string_lit":0,"string_valu":0,"stringtyp":[0,7,8],"stringvalu":0,"strip":1,"strong":1,"strongli":1,"struct":0,"structpb":0,"structur":[0,1,7,8],"studi":1,"stuff":7,"sub":[0,1,7],"sub_activ":0,"sub_ev":0,"sub_eval_parti":0,"sub_evalu":0,"sub_transpil":[],"subclass":[0,1,7,8],"sublanguag":1,"subnet":0,"subnet_id":0,"subnetfilt":1,"subnetid":1,"subnetrout":1,"subscript":1,"subsequ":[0,1],"subset":[0,8],"subsidiari":0,"subst":[0,1],"substanti":7,"substitut":0,"subtl":0,"succesfulli":1,"success":[0,1,7],"successfulli":[0,4],"suffic":1,"suffix":[0,1],"sugar":0,"suggest":1,"suit":[1,4],"suitabl":1,"sum":1,"summari":5,"sun":1,"super":[0,1],"superclass":[0,1],"supertyp":8,"supplement":0,"suppli":[0,2],"support":[0,1,2,4,8],"sure":[1,4],"surpris":4,"survei":1,"suspend":1,"suspendedprocess":1,"swallow":0,"swap":1,"symbol":2,"synopsi":5,"syntact":[0,4],"syntax":[0,1,2,7,8],"system":[0,5],"t":[0,1,2,4,7,8],"tab":0,"tabl":1,"tag":[0,7],"tag_polici":7,"tag_policy_filt":7,"tag_valu":0,"tagstatu":1,"tailor":0,"take":[0,1],"take_act":7,"taken":[1,7],"tangl":8,"target":[0,4],"target_engin":1,"targetfunc":0,"targetgroup":1,"tasktaskdefinitionfilt":1,"tb":0,"tbd":0,"team_nam":1,"tech":1,"techniqu":[],"temp":1,"templat":[0,1],"temporari":[0,8],"ten":8,"tend":0,"ters":1,"tertiari":0,"test":[0,1,2,4,7,8],"test_all_typ":0,"test_c7nlib":0,"testabl":1,"testalltyp":0,"testdata":4,"text":[0,1,2,8],"text2":0,"text_from":[0,1],"textproto":4,"textproto_to_gherkin":[],"than":[0,1,2,4,5,7],"the_activ":0,"the_filt":[0,1],"thei":[0,1,4,7,8],"them":[0,1,2],"thi":[0,1,2,3,4,5,6,7,8],"thing":[0,1,8],"think":0,"thirti":1,"those":[0,1,3,7],"three":[0,1,5,7,8],"threshold":1,"through":[0,1],"tidi":8,"tidyup":1,"time":[0,1,2,7],"timedelta":[0,1,8],"timestamp":[0,1,4,7],"timestamptyp":[0,1,7,8],"timezon":[0,1,7],"tini":2,"titl":1,"titlecas":0,"tl":1,"tlq9fr":1,"tls12":1,"tlsv1":1,"to_python":0,"todo":1,"token":0,"toler":0,"toml":[2,3],"too":[0,8],"tool":[5,6],"top":[0,1,4,8],"topic":[1,2,5],"tot":2,"total":[1,2],"toward":0,"tox":4,"tp":0,"tp1":0,"tp2":0,"tpb":0,"trace":[0,2,3],"traceback":[0,7],"tracebacktyp":0,"track":[0,7],"tradit":[0,2],"traffic":1,"trail":1,"transact":7,"transform":[0,1,4,7],"transient":[0,8],"transit":1,"translat":[0,1],"transpar":0,"transpil":[0,3,5],"transpilertre":0,"travers":1,"treat":[0,2],"tree":[0,4,7,8],"tree_class":0,"tree_dump":0,"tree_for_express":0,"tree_for_vari":0,"tree_node_class":0,"tri":1,"trigger":1,"trim":[0,1],"tripl":0,"trivial":[0,1],"troublingli":1,"true":[0,1,2,3,7],"truncat":0,"trust":1,"truthi":0,"try":[0,7],"tupl":0,"turn":[0,1],"tweet":7,"two":[0,1,2,4,7,8],"txt":[0,1],"typ":0,"type":[1,4,5],"type_match":0,"type_nam":0,"type_name_map":[],"type_schema":1,"typeerror":0,"typetyp":[0,8],"tz":[0,1,7],"tz_alias":[0,1],"tz_name":0,"tz_name_lookup":0,"tz_offset_pars":0,"tz_pars":0,"tzinfo":[0,1],"tzutc":1,"u":[0,1,7],"u0001f431":0,"u270c":0,"ubuntu":1,"uint":0,"uint32":0,"uint32valu":0,"uint64":0,"uint64_overflow_neg":0,"uint64_overflow_posit":0,"uint64_valu":[0,4],"uint64valu":0,"uint_lit":0,"uinttyp":[0,4,7,8],"uk":1,"unari":0,"unary_minus_no_overload":0,"unary_neg":0,"unary_not":0,"unchosen":0,"uncomput":7,"undecid":0,"undeclar":0,"under":1,"underli":8,"understand":8,"underutil":1,"unencrypt":1,"unexpect":4,"unifi":[0,1],"uniform":2,"union":[0,1,8],"uniqu":[0,1,8],"unique_s":[0,1],"unit":0,"unittest":0,"unknown":4,"unless":1,"unlik":0,"unmark":1,"unmodifi":1,"unoptim":1,"unsign":[0,8],"unsupport":0,"untag":1,"until":0,"unus":0,"unusediampolici":1,"unusediamrol":[0,1],"unusedinstanceprofil":[0,1],"unusedlaunchconfig":[0,1],"unusedrdssubnetgroup":[0,1],"unusedsecuritygroup":[0,1],"unusu":1,"unwieldi":1,"up":[0,1,6],"upcom":[0,1],"updat":[0,1,4],"upgradeavail":1,"upper":0,"uptimefilt":1,"uri":0,"url":[0,1,4],"us":[0,3,4,5,6,8],"usabl":[0,8],"usag":[1,2],"usediampolici":1,"usediamrol":1,"usedinstanceprofil":1,"usedsecuritygroup":1,"user":3,"user_access":2,"user_creation_tim":1,"useraccesskei":1,"userag":1,"usercredentialreport":1,"userdata":1,"userdatafilt":1,"userguid":1,"userid":2,"userident":1,"usermfadevic":1,"usernam":1,"userpolici":1,"userpolicysizequota":1,"usersquota":1,"usual":0,"utc":[0,7],"utcnow":[0,1],"util":1,"uv":[4,6],"v":[0,2,4],"val":7,"valid":[0,4],"valid_key_typ":0,"validconfigfilt":1,"valu":[2,3,4,7,8],"value_typ":0,"valueerror":0,"valuefilt":1,"valuekv":1,"values_from":1,"valuesfrom":0,"var":[0,7],"vari":1,"variabl":[0,1,3,4,5,7,8],"variant":[0,1,7],"variat":1,"varieti":[0,8],"variou":[0,1,2,8],"vault":1,"ve":[2,7],"verbos":4,"veri":[0,1,3,4,8],"verifi":1,"version":[0,1,3,4,7],"versionsperpolicyquota":1,"via":[0,1,7,8],"viabl":0,"view":8,"virtual":6,"visibl":[0,4],"visit":[0,8],"visit_children":0,"visitor":0,"visual":7,"vm":1,"vol":1,"volum":1,"volumeid":1,"volumin":2,"vpc":0,"vpc_id":0,"vpcconfig":1,"vpce":0,"vpcendpoint":1,"vpcfilter":[0,1],"vpcid":1,"vpcidfilt":1,"vpclink":1,"vpcsecuritygroupfilt":[0,1],"vpcsubnetfilt":0,"vpn":1,"w":[1,2],"wa":[0,1,7],"wafen":1,"wai":[0,1,2,7],"wait":1,"walk":[0,8],"want":[0,1,2,7],"warn":[0,1,3],"watch":1,"we":[0,1,2,7,8],"web":1,"web_acl":[0,1],"webacl":1,"week":1,"weekdai":1,"weekend":1,"weekli":1,"well":[0,1,2,7,8],"west":1,"what":[0,2,7],"when":[0,1,2,4,6,7,8],"where":[0,1,2,4,6,8],"wherea":8,"whether":[0,1,7],"which":[0,1,4,7,8],"while":[0,1,2,7,8],"whitelist":1,"whitelist_condit":1,"whitelist_endpoint":1,"whitelist_endpoints_from":1,"whitelist_from":1,"whitelist_orgid":1,"whitelist_orgids_from":1,"whitelist_protocol":1,"whitelist_protocols_from":1,"whitelist_vpc":1,"whitelist_vpc_from":1,"whitelist_vpce_from":1,"who":1,"whole":[5,7,8],"whose":1,"why":[0,2],"wide":[1,8],"width":0,"wildcard":[],"win":1,"window":[1,7],"wip":4,"wire":1,"wise":4,"with_traceback":0,"withdraw":7,"within":[0,1,7],"without":[0,1,2,3,7,8],"won":0,"work":[0,1,7,8],"workabl":1,"worker":1,"workflow":1,"world":[0,7],"woth":8,"would":[0,1,7],"wrap":[0,7,8],"wrapper":0,"wrapperspb":0,"wrire":0,"write":[2,4],"write_acp":1,"writen":1,"written":4,"wrt":1,"www":[0,1,4],"x":[0,1,2],"x_sign":0,"xrayencrypt":1,"xyz":1,"y":[0,1],"yaml":7,"ye":[1,2],"yeah":0,"yet":8,"yield":[0,1],"you":[0,1,2,7],"your":[1,4],"yyyi":1,"z":[0,1],"z0":0,"za":[0,1],"zero":[0,2,4],"zerodivisionerror":0,"zone":[0,1,7],"\u00b5":0},"titles":["API","C7N Functions Required","CLI Use of CEL-Python","Configuration","Development Tools","Pure Python Google Common Expression Language (CEL)","Installation","Application Integration","Architecture and Design"],"titleterms":{"The":[0,4,8],"__main__":0,"access":1,"account":1,"adapt":0,"addit":0,"ag":1,"alia":1,"all":1,"allow":1,"ami":1,"anoth":[],"api":0,"applic":7,"architectur":8,"argument":2,"attribut":1,"avail":1,"baselin":7,"bind":7,"block":1,"bool":1,"boolean":1,"bucket":1,"builtin":7,"bulk":7,"c7n":[0,1,7],"c7nlib":0,"cach":1,"capac":1,"cel":[2,5,8],"celfilt":1,"celpars":0,"celpi":0,"celtyp":0,"chang":[],"check":1,"cidr":1,"cli":2,"cloud":7,"cloudtrail":1,"common":[1,5],"compil":8,"complianc":1,"compon":8,"concurr":1,"config":1,"configur":[2,3],"construct":1,"contain":8,"content":[1,5],"context":[0,1,8],"convers":1,"count":1,"credenti":1,"cross":1,"custodian":7,"custom":7,"data":1,"db":1,"default":1,"defin":7,"definit":1,"delta":1,"descript":[2,4],"design":[1,8],"detail":0,"develop":4,"devic":1,"dhcp":1,"diff":1,"doc":4,"document":5,"dot":8,"eb":1,"egress":1,"enabl":1,"encrypt":1,"environ":2,"ephemer":1,"error":7,"essenti":7,"evalu":[0,8],"event":1,"exampl":[1,2,4,7],"except":7,"exit":2,"express":5,"extern":1,"fault":1,"featur":[0,4],"file":[2,4],"filter":[1,7],"find":1,"flow":1,"from":7,"function":[0,1,7],"global":[1,7],"go":7,"googl":5,"grant":1,"group":1,"ha":1,"health":1,"healthcheck":1,"i":1,"iam":1,"id":1,"imag":1,"implement":[1,7],"indic":5,"ingress":1,"inlin":1,"instal":6,"instanc":1,"integr":[1,5,7],"invalid":1,"inventori":1,"kei":1,"km":1,"languag":5,"last":1,"latest":1,"launch":1,"lifecycl":1,"limit":1,"listen":1,"locat":1,"lock":1,"log":1,"makefil":4,"manag":1,"mark":1,"member":8,"method":1,"metric":1,"mfa":1,"mismatch":1,"miss":[1,8],"modify":1,"more":7,"name":[0,8],"namespac":2,"network":1,"non":1,"note":[],"notif":1,"numer":0,"object":[0,1],"offhour":1,"onhour":1,"op":1,"oper":[],"option":1,"origin":1,"overview":5,"param":1,"paramet":1,"parser":[],"password":1,"pb2g":4,"polici":1,"principl":1,"product":8,"progag":1,"project":4,"protect":1,"protocol":1,"provid":0,"public":1,"pure":5,"py":[],"python":[2,5],"queri":1,"readm":[],"requir":1,"reserv":1,"resolut":0,"resourc":7,"rest":1,"rotat":1,"rout":1,"rule":1,"run":[],"s3":1,"secur":1,"servic":1,"shadow":1,"shield":1,"simpl":7,"singleton":1,"skip":1,"snapshot":1,"sourc":1,"specif":1,"ssl":1,"ssm":1,"stale":1,"state":1,"statement":1,"statu":[1,2],"string":[],"structur":[],"subnet":1,"summari":1,"synopsi":[2,4],"tabl":5,"tag":1,"target":1,"task":1,"termin":1,"textproto_to_gherkin":[],"time":8,"timzon":0,"todo":0,"toler":1,"tool":4,"transpil":8,"type":[0,2,7,8],"unus":1,"upgrad":1,"uptim":1,"us":[1,2,7],"user":1,"valid":1,"valu":[0,1],"value_from":[0,1],"value_typ":1,"variabl":2,"virtual":1,"vpc":1,"waf":1,"write":1,"xrai":1}}) \ No newline at end of file diff --git a/docs/build/html/structure.html b/docs/build/html/structure.html index ac5b1a0..30353c1 100644 --- a/docs/build/html/structure.html +++ b/docs/build/html/structure.html @@ -5,7 +5,7 @@ - Data Structures — CEL in Python documentation + Architecture and Design — CEL in Python documentation @@ -14,8 +14,8 @@ - - + + @@ -33,39 +33,321 @@
                  -
                  -

                  Data Structures

                  -
                  -

                  Run-Time

                  -

                  An external client depends on the celpy.Environment.

                  -

                  The celpy.Environment builds the initial AST and the final runnable “program.” -The celpy.Environment may also contain a type provider and type adapters.

                  -

                  The celpy.Environment also builds -an celpy.evaluation.Activation with the variable and function bindings -and the default package.

                  -

                  The celpy.evaluation.Activation create a kind of chainmap for name -resolution. The chain has the following structure:

                  +
                  +

                  Architecture and Design

                  +

                  We’ll start with the C4 views:

                  +
                    +
                  • Context

                  • +
                  • Container – this isn’t too interesting, but it can help to see this.

                  • +
                  • Components

                    +

                    This is a collection of various design notes describing some implementation details.

                      -
                    • The end of the chain is the built-in defaults.

                    • -
                    • A layer on top of this can be provided as part of integration into some other app or framework.

                    • -
                    • The next layer is the “current” activation when evaluating a given expression. -This often has command-line variables.

                    • -
                    • A transient top-most layer is used to create a local variable binding -for the macro evaluations.

                    • +
                    • Compile-Time

                    • +
                    • Evaluation-Time

                    • +
                    • CEL Types

                    • +
                    • Transpiler Missing Names

                    • +
                    • The member-dot Production

                    • +
                    +
                  • +
                  +

                  The code view is in the API section.

                  +
                  +

                  Context

                  +

                  There are two distinct contexts for CEL Python:

                  +
                    +
                  • The CLI – as a stand-alone application.

                  • +
                  • As an importable module to provide expressions to a DSL.

                  • +
                  +

                  +@startuml
+skinparam actorStyle awesome
+left to right direction
+
+package celpy {
+    package library {
+    usecase lib1 as "extend DSL with expressions"
+    usecase lib2 as "create program"
+    usecase lib3 as "evaluate program in context"
+    lib1 --> lib2
+    lib1 --> lib3
+    }
+
+    package cli {
+    usecase cli1 as "**expr** features
+    ---
+    Use the -n option"
+    usecase cli2 as "**test** features
+    ---
+    Use the -nb options"
+    usecase cli3 as "**jq** features
+    Newline-Delimited or single JSON doc"
+    usecase cli4 as "interactive computation
+    ---
+    use the -i option"
+    }
+
+}
+
+actor app as "some app with a DSL"
+app --> lib1
+
+actor bash as "shell script"
+bash --> cli1
+bash --> cli2
+bash --> cli3
+
+actor user
+user --> cli4
+
+app <|- [c7n]
+@enduml +

                  +

                  From the CLI, the celpy application has a number of use cases:

                  +
                    +
                  • A shell script can use celpy as a command to replace other shell commands, including expr, test, and jq.

                  • +
                  • A person can run celpy interactively. +This allows experimentation. +It also supports exploring very complex JSON documents to understand their structure.

                  • +
                  +

                  As a library, an application (for example, C7N) can import celpy to provide an expression feature for the DSL. +This provides well-defined semantics, and widely-used syntax for the expression language. +There’s an explicit separation between building a program and executing the program to allow caching an expression for multiple executions without the overhead of building a Lark parser or compiling the expression.

                  +
                  +
                  +

                  Container

                  +

                  As a CLI, this is part of a shell script. It runs where the script runs.

                  +

                  As a library, this is improted into the application to extend the DSL.

                  +

                  There are no services offered or used.

                  +
                  +
                  +

                  Components

                  +

                  The Python code base has a number of modules.

                  +
                    +
                  • __init__ – the celpy package as a whole.

                  • +
                  • __main__ – the main applications used when running celpy.

                  • +
                  • celparser – a Facade for the Lark parser.

                  • +
                  • evaluation – a Facade for run-time evaluation.

                  • +
                  • celtypes – the underlying Python implementations of CEL data structures.

                  • +
                  • c7nlib– a collection of components the C7N can use to introduce CEL filters.

                  • +
                  • adapter – Some JSON serialization components.

                  • +
                  +

                  Here’s the conceptual organiation

                  +

                  +@startuml
+
+package celpy {
+    component "~__init__" as init
+    component "~__main__" as main
+    component adapter
+    component c7nlib
+    component celparser
+    component celtypes
+    component evaluation
+    component cel.lark
+}
+init --> celtypes
+init --> adapter
+init --> celparser
+init--> evaluation
+
+main --> init
+main --> celparser
+main --> adapter
+main --> evaluation
+
+adapter --> celtypes
+
+c7nlib --> evaluation
+c7nlib --> adapter
+c7nlib --> celtypes
+c7nlib --> init
+
+celparser --> cel.lark
+celparser --> lark
+
+evaluation --> lark
+evaluation --> celtypes
+
+package lark {
+}
+@enduml +

                  +

                  While there is a tangle of dependencies, there are three top-level “entry points” for celpy.

                  +
                    +
                  • The __main__ module is the CLI application.

                  • +
                  • The c7nlib module exposes CEL functionality in a form usable by Cloud Custodian filter definitions. +This library provides useful components to perform Custodian-related computations.

                  • +
                  • The __init__ module is exposes the most useful parts of celpy for integration woth another application.

                  • +
                  +
                  +

                  Compile-Time

                  +

                  Here are the essential classes used to compile a CEL expression and prepare it for evaluation.

                  +

                  +@startuml
+hide empty members
+
+class Environment {
+    package: str
+    annotations: dict[str, Annotation]
+    compile(text: str) -> lark.Tree
+    program(expr: lark.Tree, functions: dict) -> Runner
+}
+
+class celparser.CELParser{
+    parse(text: str)
+}
+Environment *-- CELParser
+
+class lark.Tree {}
+CELParser --> lark.Tree : "Creates"
+
+abstract class Runner {
+    ast: Tree
+    evaluate(context: Context) -> Value
+}
+Environment --> Runner : "Creates"
+Runner o-- lark.Tree
+Runner o-- "0..m" CELFunction
+
+class InterpretedRunner
+Runner <|-- InterpretedRunner
+
+class evaluation.Evaluator
+InterpretedRunner *-- Evaluator
+
+class CompiledRunner
+Runner <|-- CompiledRunner
+
+class evaluation.Transpiler
+CompiledRunner *-- Transpiler
+
+class evaluation.Context << (T,orchid) Type>> {
+    key: str
+    value: Result | NameContainer
+}
+Runner o--- "0..m" Context
+
+class CELFunction <<Callable>>
+
+class Annotation << (T,orchid) Type>>
+Environment o-- "0..m" Annotation
+
+class TypeType
+Annotation <|-- TypeType
+Annotation <|-- CELFunction
+
+@enduml +

                  +

                  The fundamental sequence of operations is

                  +
                    +
                  1. Create an celpy.Environment with any needed celpy.Annotation instances. +For the most part, these are based on the overall application domain. +Any type definitions should be subclasses of celpy.TypeType or a callable function defined by the celpy.CELFunction type.

                  2. +
                  3. Use the celpy.Environment to compile the CEL text to create a parse tree.

                  4. +
                  5. Use the celpy.Environment to create a celpy.Runner instance from the parse tree and any function definitions that override or extend the predefined CEL environment.

                  6. +
                  7. Evaluate the celpy.Runner with a celpy.Context. +The celpy.Context provides specific values for variables required for evaluation. +Generally, each variable should have an celpy.Annotation defined in the celpy.Environment.

                  8. +
                  +

                  The celpy.Runner can be evaluated with any number of distinct celpy.Context values. +This amortizes the cost of compilation over multiple executions.

                  +
                  +
                  +

                  Evaluation-Time

                  +

                  Here’s the classes to evaluate a CEL expression.

                  +

                  +@startuml
+hide empty members
+
+abstract class Runner {
+    ast: Tree
+    evaluate(context: Context) -> Value
+}
+Environment --- Runner : "Created By <"
+Runner o-- "0..m" CELFunction
+Runner o-- Context
+
+class lark.Tree
+Tree --* Runner
+
+class InterpretedRunner <<Adapter>>
+Runner <|-- InterpretedRunner
+
+abstract class lark.Interpreter
+
+class evaluation.Evaluator {
+    activation: Activation
+    functions: dict[str, CELFunction]
+    evaluate() -> Value
+}
+lark.Interpreter <|--- Evaluator
+InterpretedRunner *-- Evaluator
+
+class CompiledRunner  <<Adapter>>
+Runner <|-- CompiledRunner
+
+InterpretedRunner -[hidden]> CompiledRunner
+
+class evaluation.Transpiler {
+    functions: dict[str, CELFunction]
+    transpile()
+    evaluate() -> Value
+}
+CompiledRunner *-- Transpiler
+lark.Interpreter <|--- Transpiler
+
+class evaluation.Activation {
+    annotations: Annotation
+    identifiers: dict[str, Result | CELFunction]
+}
+Runner *-- Activation : "Uses"
+Runner --> Activation : "Creates"
+Activation --> Activation : "based on"
+
+class Annotation << (T,orchid) Type>>
+Runner *-- "0..m" Annotation
+Annotation --o Activation : Initializes
+CELFunction --o Activation : Initializes
+Context --o Activation : Initializes
+
+@enduml +

                  +

                  The evalation of the CEL expression is done via a celpy.Runner object. +There are two celpy.Runner implementations.

                  +
                    +
                  • The celpy.InterpretedRunner walks the AST, creating the final result celpy.Value or celpy.CELEvalError exception. +This uses a celpy.evaluation.Activation to perform the evaluation.

                  • +
                  • The celpy.CompiledRunner transpiles the AST into a Python sequence of statements. +The internal compile() creates a code object that can then be evaluated with a given celpy.evaluation.Activation +The internal exec() functions performs the evaluation.

                  -

                  The AST is created by Lark from the CEL expression.

                  -

                  There are two celpy.Runner implementations.

                  +

                  The subclasses of celpy.Runner are Adapter classes to provide a tidy interface to the somewhat more complex celpy.Evaluator or celpy.Transpiler objects. +In the case of the celpy.InterpretedRunner, evaluation involves creating an celpy.evaluation.Activation and visiting the AST. +Whereas, the celpy.CompiledRunner must first visit the AST to create code. At evaluation time, it create an celpy.evaluation.Activation and uses exec() to compute the final value.

                  +

                  The celpy.evaluation.Activation contains several things:

                    -
                  • The celpy.InterpretedRunner walks the AST, creating the final result or exception.

                  • -
                  • The celpy.CompiledRunner transforms the AST to remove empty rules. Then emits -the result as a Python expression. It uses the Python internal compile() and eval() functions -to evaluate the expression.

                  • +
                  • The Annotation definitions to provide type information for identifiers.

                  • +
                  • The CELFunction functions that extend or override the built-in functions.

                  • +
                  • The values for identifiers.

                  • +
                  +

                  The celpy.evaluation.Activation is a kind of chainmap for name resolution. +The chain has the following structure:

                  +
                    +
                  • The end of the chain has the built-in defaults. +(This is the bottom-most base definition.)

                  • +
                  • A layer on top of this can offer types and functions which are provided to integrate into the containing app or framework.

                  • +
                  • The next layer is the “current” activation when evaluating a given expression. +For the CLI, this has the command-line variables. +For other integrations, these are the input values.

                  • +
                  • A transient layer on top of this is used to create a local variable binding for the macro evaluations. +These can be nested, and introduce the macro variable as a temporary annotation and value binding.

                  -

                  CEL Types

                  +

                  CEL Types

                  There are ten extension types that wrap Python built-in types to provide the unique CEL semantics.

                    +
                  • celtypes.TypeType is a supertype for CEL types.

                  • celtypes.BoolType wraps int and creates additional type overload exceptions.

                  • celtypes.BytesType wraps bytes it handles conversion from celtypes.StringType.

                  • celtypes.DoubleType wraps float and creates additional type overload exceptions.

                  • @@ -81,7 +363,69 @@

                    CEL Typesdatetime.timedelta, int, and str values.

                  Additionally, a celtypes.NullType is defined, but does not seem to be needed. It hasn’t been deleted, yet. -but should be considered deprecated.

                  +It should be considered deprecated.

                  +
                  +
                  +
                  +

                  Transpiler Missing Names

                  +

                  The member_dot transpilation with a missing name will be found at evaluation time via member.get('IDENT'). This raises No Such Member in Mapping error.

                  +

                  The primary :: ident evaluation can result in one of the following conditions:

                  +
                  +
                    +
                  • ident denotes a type definition. The value’s type is TypeType. +The value is a type reference bool becomes celpy.celtypes.BoolType.

                  • +
                  • ident denotes a built-in function. The value’s type is CELFunction. +The value is the Python function reference.

                  • +
                  • ident denotes an annotation, but the value’s type is neither TypeType nor CELFunction.

                    +

                    The transpiled value is f"activation.{ident}", assuming it will be a defined variable.

                    +

                    If, at exec() time the name is not in the Activation with a value, a NameError exception will be raised that becomes a CELEvalError exception.

                    +
                  • +
                  +
                  +
                  +
                  +

                  The Member-Dot Production

                  +

                  Consider protobuf_message{field: 42}.field. +This is parsed using the following productions.

                  +
                  member         : member_dot | member_dot_arg | member_item | member_object | primary
                  +member_dot     : member "." IDENT
                  +member_object  : member "{" [fieldinits] "}"
                  +
                  +
                  +

                  The member_object will be a primary which can be an ident. +It MUST refer to the Annotation (not the value) because it has fieldinits. +All other choices are (generally) values. +They can be annotations, which means bool.type() works the same as type(bool).

                  +

                  Here’s primary production, which defines the ident in the member production.

                  +
                  primary        : dot_ident_arg | dot_ident | ident_arg | ident
                  +               | paren_expr | list_lit | map_lit | literal
                  +
                  +
                  +

                  The ident is not always transpiled as activation.{name}. +Inside member_object, it’s activation.resolve_name({name}). +Outside member_object, it can be activation.{name} because it’s a simple variable.

                  +

                  It may make sense to rename the Activation.resolve_name() method to Activation.get().

                  +

                  This, however, overloads the get() method. +This has type hint consequences.

                  +
                  +

                  Important

                  +

                  The member can be any of a variety of objects:

                  +
                    +
                  • NameContainer(Dict[str, Referent])

                  • +
                  • Activation

                  • +
                  • MapType(Dict[Value, Value])

                  • +
                  • MessageType(MapType)

                  • +
                  +

                  All of these classes must define a get() method. +The nuance is the NameContainer is also a Python dict and there’s an +overload issue between that get() and other get() definitions.

                  +
                  +

                  The Transpilation currently leverages a common method named get() for all of these types. +This is a Pythonic approach, but, the overload for the NameContainer (a Dict subclass) isn’t quite right: +it doesn’t return a Referent, but the value from a Referent.

                  +

                  A slightly smarter approach is to define a get_value(member, 'name') function that uses a match/case structure to do the right thing for each type. The problem is, the result is a union of type, value, function, and any of these four containers!

                  +

                  Another possibility is to leverage the Annotations. +They can provide needed type information to discern which method with specific result type.

                  @@ -111,18 +455,22 @@

                  CEL in Python

                  Navigation

                  -

                  Contents:

                  +

                  Documentation Content:

                  diff --git a/docs/plantuml-asl-1.2025.3.jar b/docs/plantuml-asl-1.2025.3.jar new file mode 100644 index 0000000..2faf142 Binary files /dev/null and b/docs/plantuml-asl-1.2025.3.jar differ diff --git a/docs/source/api.rst b/docs/source/api.rst index da47f98..7515b4f 100644 --- a/docs/source/api.rst +++ b/docs/source/api.rst @@ -5,11 +5,30 @@ .. _`api`: ########### -CEL-Py API +API ########### Details of the CEL-Python implementation and the API to the various components. +The following modules are described here: + +- celpy_ (The ``__init__.py`` file defines this.) + +- \_\_main\_\_ (The CLI main application.) + +- adapter_ + +- c7nlib_ + +- celparser_ + +- celtypes_ + +- evaluation_ + +``celpy`` +========= + .. automodule:: celpy.__init__ ``__main__`` @@ -17,18 +36,28 @@ Details of the CEL-Python implementation and the API to the various components. .. automodule:: celpy.__main__ -celtypes -========= +``adapter`` +=========== -.. automodule:: celpy.celtypes +.. automodule:: celpy.adapter -evaluation -========== +``c7nlib`` +=========== -.. automodule:: celpy.evaluation +.. automodule:: celpy.c7nlib -parser -====== +``celparser`` +============= .. automodule:: celpy.celparser +``celtypes`` +============= + +.. automodule:: celpy.celtypes + +``evaluation`` +============== + +.. automodule:: celpy.evaluation + diff --git a/docs/source/c7n_functions.rst b/docs/source/c7n_functions.rst index 62b8c6d..edc12bd 100644 --- a/docs/source/c7n_functions.rst +++ b/docs/source/c7n_functions.rst @@ -5,15 +5,13 @@ C7N Functions Required ###################### -This survey of C7N filter clauses is based on source code and -on an analysis of working policies. The required functions -are grouped into four clusters, depending on the presence of absence of -the "op" operator clause, and the number of resource types using the feature. +This survey of C7N filter clauses is based on source code and on an analysis of working policies. +The required functions are grouped into four clusters, depending on the presence of absence of the "op" operator clause, and the number of resource types using the feature. Within each group, they are ranked in order of popularity. For each individual type of filter clause, we provide the following details: -- The C7N sechema definition. +- The C7N schema definition. - The resource types where the filter type is used. @@ -43,7 +41,7 @@ interface. 1. Separate from C7N. The CEL processing is outside C7N, and capable of standing alone. CEL is focused on Protobuf (and JSON) objects. The interface to C7N is via the :mod:`c7nlib` library of functions. These do **not** depend - on imports from the C7N project, but rely on a `CELFilter` class offering specific methods. + on imports from the C7N project, but rely on a ``CELFilter`` class offering specific methods. Access to C7N objects and their associated methods is limited to the features exposed through the function library and the expected class definition. diff --git a/docs/source/cli.rst b/docs/source/cli.rst index 3cb4b02..96a5319 100644 --- a/docs/source/cli.rst +++ b/docs/source/cli.rst @@ -6,50 +6,302 @@ CLI Use of CEL-Python ###################### -We can read JSON directly from stdin, making this a bit like JQ. +While CEL-Python's primary use case is integration into an DSL-based application to provide expressions with a uniform syntax and well-defined semantics. +The expression processing capability is also available as a CLI implemented in the ``celpy`` package. + +SYNOPSIS +======== :: - % PYTHONPATH=src python -m celpy '.this.from.json * 3 + 3' <, --arg + + Define argument variables, types, and (optional) values. + If the argument value is omitted, then an environment variable will be examined to find the value. + For example, ``--arg HOME:string`` makes the :envvar:`HOME` environment variable's value available to the CEL expression. + +.. option:: -b, --boolean + + Return a status code value based on the boolean output. + + true has a status code of 0 + + false has a statis code of 1 + + Any exception has a stats code of 2 + +.. option:: -n, --null-input + + Do not read JSON input from stdin + +.. option:: -s, --slurp + + Treat all input as a single JSON document. + The default is to treat each line of input as a separate NLJSON document. + +.. option:: -i, --interactive + + Operate interactively from a ``CEL>`` prompt. + In :option:`-i` mode, the rest of the options are ignored. + +.. option:: -p, --json-package + + Each NDJSON input (or the single input in :option:`-s` mode) + is a CEL package. + +.. option:: -d, --json-document + + Each NDJSON input (or the single input in :option:`-s` mode) + is a separate CEL variable. + +.. option:: -f , --format + + Use Python formating instead of JSON conversion of results; + Example ``--format .6f`` to format a ``DoubleType`` result + +.. option:: expr + + A CEL expression to evaluate. + +DESCRIPTION +============ + +This provides shell-friendly expression processing. +It follows patterns from several programs. + +:jq: + The ``celpy`` application will read newline-delimited JSON + from stdin. + It can also read a single, multiline JSON document in ``--slurp`` mode. + + This will evaluate the expression for each JSON document. + + .. note:: + + ``jq`` uses ``.`` to refer the current document. By setting a package + name of ``"jq"`` with the :option:`-p` option, e.g., ``-p jq``, + and placing the JSON object in the same package, we achieve + similar syntax. + +:expr: + The ``celpy`` application does everything ``expr`` does, but the syntax is different. + + The output of comparisons in ``celpy`` is boolean, where by default. + The ``expr`` program returns an integer 1 or 0. + Use the :option:`-f` option, for example, ``-f 'd'`` to see decimal output instead of Boolean text values. + +:test: + This does what ``test`` does using CEL syntax. + The ``stat()`` function retrieves a mapping with various file status values. + + Use the :option:`-b` option to set the exit status code from the Boolean result. + + A ``true`` value becomes a 0 exit code. + + A ``false`` value becomes a 1 exit code. + +:bc: + THe little-used linux ``bc`` application has several complex function definitions and other programming support. + CEL can evaluate some ``bc``\\ -like expressions. + It could be extended to mimic ``bc``. + +Additionally, in :option:`--interactive` mode, +there's a REPL with a ``CEL>`` prompt. + +Arguments, Types, and Namespaces +--------------------------------- + +The :option:`--arg` options must provide a variable name and type. +CEL objects rely on the :py:mod:`celpy.celtypes` definitions. + +Because of the close association between CEL and protobuf, some well-known protobuf types +are also supported. + +The value for a variable is optional. +If it is not provided, then the variable is presumed to be an environment variable. +While many environment variables are strings, the type is still required. +For example, use ``--arg HOME:string`` to get the value of the :envvar:`HOME` environment variable. + +FILES +====== + +By default, JSON documents are read from stdin in NDJSON format (http://jsonlines.org/, http://ndjson.org/). +For each JSON document, the expression is evaluated with the document in a default +package. This allows `.name` to pick items from the document. + +By default, the output is JSON serialized. +This means strings will be JSON-ified and have quotes. +Using the :option:`-f` option will expect a single, primitive type that can be formatting using Python's string formatting mini-language. + +ENVIRONMENT VARIABLES +===================== + +Enhanced logging is available when :envvar:`CEL_TRACE` is defined. +This is quite voluminous; tracing most pieces of the AST during evaluation. + +CONFIGURATION +============= + +Logging configuration is read from the ``celpy.toml`` file. +See :ref:`configuration` for details. + +EXIT STATUS +=========== + +Normally, it's zero. + +When the :option:`-b` option is used then the final expression determines the status code. + +A value of ``true`` returns 0. + +A value of ``false`` returns 1. + +Other values or an evaluation error exception will return 2. + +EXAMPLES +======== + +We can read JSON directly from stdin, making this a bit like the **jq** application. +We provide a JQ expression, ``'.this.from.json * 3 + 3'``, and a JSON document. +The standard output is the computed result. + +.. code-block:: bash + + % python -m celpy '.this.from.json * 3 + 3' < {"this": {"from": {"json": 13}}} heredoc> EOF 42 +The default behavior is to read and process stdin, where each line is a separate JSON document. +This is the Newline-Delimited JSON format. +(See https://jsonlines.org and https://github.com/ndjson/ndjson-spec). + +The ``-s/--slurp`` treats the stdin as a single JSON document, spread over multiple lines. +This parallels the way the the **jq** application handles JSON input. + +We can avoid reading stdin by using the ``-n/--null-input`` option. +This option will evaluate the expression using only command-line argument values. + It's also a desk calculator. -:: +.. code-block:: bash % python -m celpy -n '355.0 / 113.0' 3.1415929203539825 -And, yes, this has a tiny advantage over ``python -c '355/113'``. Most notably, the ability -to embed Google CEL into other contexts where you don't *really* want Python's power. -There's no CEL ``import`` or built-in ``exec()`` function to raise concerns. +And, yes, this use case has a tiny advantage over ``python -c '355/113'``. +Most notably, the ability to embed Google CEL into other contexts where you don't *really* want Python's power. +There's no CEL ``import`` or built-in ``eval()`` function to raise security concerns. -We can provide a ``-d`` option to define objects with particular data types, like JSON. -This is particularly helpful for providing protobuf message definitions. +We can provide a ``-a/--arg`` option to define a name in the current activation with particular data type. +The expression, ``'x * 3 + 3'`` depends on a ``x`` variable, set by the ``-a`` option. +Note the ``variable:type`` syntax for setting the type of the variable. -:: +.. code-block:: bash - % PYTHONPATH=src python -m celpy -n -ax:int=13 'x * 3 + 3' + % python -m celpy -n -ax:int=13 'x * 3 + 3' 42 -This command sets a variable ``x`` then evaluates the expression. And yes, this is what -``expr`` does. CEL can do more. For example, floating-point math. +This is what the bash ``expr`` command does. +CEL can do more. +For example, floating-point math. +Here we've set two variables, ``x`` and ``tot``, before evaluating an expression. -:: +.. code-block:: bash - % PYTHONPATH=src python -m celpy -n -ax:double=113 -atot:double=355 '100. * x/tot' + % python -m celpy -n -ax:double=113 -atot:double=355 '100. * x/tot' 31.830985915492956 -We can also mimic the ``test`` command. +If you omit the ``=`` from the ``-a`` option, then an environment variable's +value will be bound to the variable name in the activation. -:: +.. code-block:: bash + + % TOTAL=41 python -m celpy -n -aTOTAL:int 'TOTAL + 1' + 42 + +Since these operations involves explict type conversions, be aware of the possibility of syntax error exceptions. + +.. code-block:: bash - % PYTHONPATH=src python -m celpy -n -ax:int=113 -atot:int=355 -b 'x > tot' + % TOTAL="not a number" python -m celpy -n -aTOTAL:int 'TOTAL + 1' + usage: celpy [-h] [-v] [-a ARG] [-n] [-s] [-i] [--json-package NAME] [--json-document NAME] [-b] [-f FORMAT] [expr] + celpy: error: argument -a/--arg: arg TOTAL:int value invalid for the supplied type + + + +We can also use this instead of the bash ``test`` command. +We can bind values with the ``-a`` options and then compare them. +The ``-b/--boolean`` option sets the status value based on the boolean result value. +The output string is the CEL literal value ``false``. +The status code is a "failure" code of 1. + +.. code-block:: bash + + % python -m celpy -n -ax:int=113 -atot:int=355 -b 'x > tot' false % echo $? 1 -The intent is to provide a common implementation for aritmetic and logic. +Here's another example that shows the ``stat()`` function to get filesystem status. + +.. code-block:: bash + + % python -m celpy -n -aHOME 'HOME.stat()' + {"st_atime": "2025-07-06T20:27:21Z", "st_birthtime": "2006-11-27T18:30:03Z", "st_ctime": "2025-07-06T20:27:20Z", "st_dev": 16777234, "st_ino": 341035, "st_mtime": "2025-07-06T20:27:20Z", "st_nlink": 135, "st_size": 4320, "group_access": true, "user_access": true, "kind": "d", "setuid": false, "setgid": false, "sticky": false, "r": true, "w": true, "x": true, "st_blksize": 4096, "st_blocks": 0, "st_flags": 0, "st_rdev": 0, "st_gen": 0} + +As an example, to compare modification time between two files, use an expression like ``f1.stat().st_mtime < f2.stat().st_mtime``. + +This is longer than the traditional bash expression, but much more clear. + +The file "kind" is a one-letter code: +:b: block +:c: character-mode +:d: directory +:f: regular file +:p: FIFO or pipe +:l: symbolic link +:s: socket + +The ``r``, ``w``, and ``x`` attributes indicate if the current effective userid can read, write, or execute the file. This comes from the detailed permission bits. + +The intent is to provide a single, uniform implementation for arithmetic and logic operations. +The primary use case integration into an DSL-based application to provide expressions without the mental burden of writing the parser and evaluator. + +We can also use CEL interactively, because, why not? + +.. code-block:: bash + + % python -m celpy -i + Enter an expression to have it evaluated. + CEL> 355. / 113. + 3.1415929203539825 + CEL> ? + + Documented commands (type help ): + ======================================== + bye exit help quit set show + + CEL> help set + Set variable expression + + Evaluates the expression, saves the result as the given variable in the current activation. + + CEL> set a 6 + 6 + CEL> set b 7 + 7 + CEL> a * b + 42 + CEL> show + {'a': IntType(6), 'b': IntType(7)} + CEL> bye + % + +The ``bye``, ``exit``, and ``quit`` commands all exit the application. diff --git a/docs/source/conf.py b/docs/source/conf.py index 680767b..7009f0b 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -16,7 +16,7 @@ # import os # import sys # sys.path.insert(0, os.path.abspath('.')) - +from pathlib import Path # -- Project information ----------------------------------------------------- @@ -34,6 +34,7 @@ 'sphinx.ext.autodoc', 'sphinx.ext.viewcode', 'sphinx.ext.todo', + 'sphinxcontrib.plantuml', ] # Add any paths that contain templates here, relative to this directory. @@ -66,3 +67,8 @@ 'special-members': True, 'exclude-members': '__weakref__,__module__,__dict__,__annotations__,__slots__' } + +# -- Options for PlantUML + +DOCS = Path.cwd().parent +plantuml = f'java -jar {DOCS/"plantuml-asl-1.2025.3.jar"!s}' diff --git a/docs/source/configuration.rst b/docs/source/configuration.rst index e029b5d..93bf760 100644 --- a/docs/source/configuration.rst +++ b/docs/source/configuration.rst @@ -2,25 +2,13 @@ # Copyright 2020 The Cloud Custodian Authors. # SPDX-License-Identifier: Apache-2.0 +.. _configuration: + ###################### Configuration ###################### -The CLI application can bind argument values from the environment. -The command-line provides variable names and type information. -The OS environment provides string values. - -.. code:: bash - - export x=6 - export y=7 - celpy -n --arg x:int --arg y:int 'x*y' - 42 - -While this example uses the OS environment, -it isn't the usual sense of *configuration*. -The only configuration options available for the command-line application are the logging configuration. - +The **celpy** package uses a configuration file to set the logging options. If a ``celpy.toml`` file exists in the local directory or the user's ``HOME`` directory, this will be used to provide logging configuration for the ``celpy`` application. This file must have a ``logging`` paragraph. @@ -40,11 +28,41 @@ This paragraph can contain the parameters for logging configuration. class = "logging.StreamHandler" formatter = "console" +This provides minimal log output, showing only warnings, errors, and fatal error messages. +The ``root.level`` needs to be "INFO" or "DEBUG" to see more output. +Setting a specific logger's level to "DEBUG" will raise the logging level for a specific component. + +All of the **celpy** loggers have names starting with ``celpy.``. +This permits integration with other application without polluting those logs with **celpy** output. + To enable very detailed debugging, do the following: - Set the ``CEL_TRACE`` environment variable to some non-empty value, like ``"true"``. This enables a ``@trace`` decorator on some evaluation methods. -- In a ``[logging.loggers.celpy.Evaluator]`` paragraph, set ``level = "DEBUG"``. +- Add a ``[logging.loggers.celpy.Evaluator]`` paragraph, with ``level = "DEBUG"``. + This can be done for any of the ``celpy`` components with loggers. + +- In the ``[logging]`` paragraph, set ``root.level = "DEBUG"``. + +Loggers include the following: + +- ``celpy`` + +- ``celpy.Runner`` + +- ``celpy.Environment`` + +- ``celpy.repl`` + +- ``celpy.c7nlib`` + +- ``celpy.celtypes`` + +- ``celpy.evaluation`` + +- ``celpy.NameContainer`` + +- ``celpy.Evaluator`` -- Set the ``[logging]`` paragraph, set ``root.level = "DEBUG"``. +- ``celpy.Transpiler`` diff --git a/docs/source/development.rst b/docs/source/development.rst new file mode 100644 index 0000000..02a00f3 --- /dev/null +++ b/docs/source/development.rst @@ -0,0 +1,288 @@ +.. comment + # Copyright 2025 The Cloud Custodian Authors. + # SPDX-License-Identifier: Apache-2.0 + +###################### +Development Tools +###################### + +The development effort is dependent on several parts of the CEL project. + +1. The language specification. https://github.com/google/cel-spec/blob/master/doc/langdef.md + +2. The test cases. https://github.com/google/cel-spec/tree/master/tests/simple/testdata + +The language specification is transformed into a Lark grammar. +This is in the ``src/cel.lark`` file. +This changes very slowly. +Any changes must be reflected (manually) by revising the lark version of the EBNF. + +The test cases present a more challenging problem. + +A tool, ``pb2g.py``, converts the test cases from Protobuf messages to Gherkin scenarios. + +.. uml:: + + @startuml + + file source as "source protobuf test cases" + file features as "Gherkin feature files" + + source --> [pb2g.py] + [pb2g.py] --> [Docker] : "Uses" + [Docker] ..> [mkgherkin.go] : "Runs" + [pb2g.py] --> features + @enduml + + +The pb2g Tool +============== + +The ``pb2g.py`` Python application converts a protobuf test case collection into a Gherkin Feature file. +These can be used to update the ``features`` directory. + +SYNOPSIS +--------- + +.. program:: python tools/pb2g.py [-g docker|local] [-o output] [-sv] source + +.. option:: -g , --gherkinizer , --output + + Where to write the feature file. + Generally, it's helpful to have the ``.textproto`` and ``.feature`` stems match. + The ``Makefile`` assures this. + +.. option:: -s, --silent + + No console output is produced + +.. option:: -v, --verbose + + Verbose debugging output on the console. + +.. option:: source + + A source ``.textproto`` file. + This is often the path to a file in a local download of https://github.com/google/cel-spec/tree/master/tests/simple/testdata. + + A URL for the source is **not** supported. + + +DESCRIPTION +----------- + +Convert one ``.textproto`` file to a Gherkin ``.feature`` file. +There are two steps to the conversion: + +1. Rewrite the ``.textproto`` into JSON. + This relies on common Go libraries, and is little more than a syntactic conversion. + +2. Rewrite the JSON copy of the ``.textproto`` into Gherkin. + This a little more fraught with special cases and exceptions. + The ``.textproto`` semantics can be confusing. + +FILES +----- + +:source: + A ``.textproto`` test case file from the CEL-spec repository. + +:output: + A ``.feature`` file with the same stem as the source file is written to the output directory. + ``basic.textproto`` will create ``basic.feature``. + +:interim: + An interim JSON-format file is created and deleted. + These are only visible in the event of a fatal error creating the Gherkin output. + +EXAMPLES +-------- + +The ``basic.textproto`` starts like this: + +.. code-block:: protobuf + + name: "basic" + description: "Basic conformance tests that all implementations should pass." + section { + name: "self_eval_zeroish" + description: "Simple self-evaluating forms to zero-ish values." + test { + name: "self_eval_int_zero" + expr: "0" + value: { int64_value: 0 } + } + test { + name: "self_eval_uint_zero" + expr: "0u" + value: { uint64_value: 0 } + } + +The Feature file created looks like this: + +.. code-block:: gherkin + + Feature: basic + Basic conformance tests that all implementations should pass. + + # self_eval_zeroish -- Simple self-evaluating forms to zero-ish values. + + Scenario: self_eval_int_zero + + When CEL expression "0" is evaluated + # int64_value:0 + Then value is IntType(source=0) + + + Scenario: self_eval_uint_zero + + When CEL expression "0u" is evaluated + # uint64_value:0 + Then value is UintType(source=0) + +The source files have a "section" heading which doesn't have a precise parallel in the Gherkin language. +The sections become comments in the Feature file. + +The ``features/steps`` directory has step definition modules that implement the ``Given``, ``When``, and ``Then`` clauses. + +.. py:module:: features.steps.c7n_integration + + Provides step definitions for the ``c7n_interface.feature``. + This is not part of the CEL language specification. + +.. py:module:: features.steps.cli_binding + + Provides step definitions for the ``expr_test_bc.feature``, ``json_query.feature``, neither of which are part of the CEL language specificaiton. + +.. py:module:: features.steps.integration_binding + + Provides step definitions for the features generated by the ``pb2g.py`` tool. + +The ``features/Makefile`` +========================= + +This Makefile has the following targets: + +:%.textproto: + This copies textproto files from the source directory + to the ``features`` directory. + The source is defined by the :envvar:`CEL_SIMPLE_TESTDATA` environment variable. + This will overwrite out-of-date files in the ``features`` directory. + + It's important to use **git** wisely and start with a clean branch of the project so changes can be rolled back. + +:%.feature: + This creates the ``.feature`` file from the ``.textproto`` file. + +:scan: + This phony target reads **all** of the ``.textproto`` files to be sure they can be converted to Gherkin. + If it concludes with the output ``"All files scanned successfully"``, then there are no surprising or unexpected features in the ``.textproto`` files. + +:clean-broken: + This phony target removes empty ``.feature`` files that may be left over when the conversion process crashes with a fatal error. + +:clean-features: + This phony target removes all of the ``.textproto``\ -based ``.feature`` files. + Manually created ``.feature`` files are left intact. + +:clean: + This phony target removes all ``.textproto`` and ``.feature`` files that are built from the CEL specification. + Manually created ``.feature`` files are left intact. + +Currently, the following feature files are built from the CEL specification. + +.. code-block:: bash + + basic.feature + comparisons.feature + conversions.feature + dynamic.feature + enums.feature + fields.feature + fp_math.feature + integer_math.feature + lists.feature + logic.feature + macros.feature + namespace.feature + parse.feature + plumbing.feature + proto2.feature + proto3.feature + string.feature + timestamps.feature + unknowns.feature + +The ``docs/Makefile`` +===================== + +This is a Sphinx ``Makefile`` to build documentation. +For more information, see https://www.sphinx-doc.org/en/master/index.html + +The Project ``Makefile`` +========================= + +A top-level Makefile has a number of phony targets: + +:build: + Runs ``uv build`` to create a distribution kit. + +:install-tools: + Pulls a ``golang`` Docker image and builds the ``mkgherkin`` image. + +:test: + Runs the Python 3.12 test environment to execute a quick test. + +:test-all: + Update the ``features`` files and run the full test suite. + +:test-wip: + Update the ``features`` files and run the WIP test environment -- these are tests flagged with @WIP markers. + +:test-tools: + Run a test of only the tools, then scan the ``features`` files to be sure they're still valid after the tool change. + +:docs: + Build the HTML documentation. + +:lint: + Runs the ``lint`` test environment to get code coverage, type hint checking, and other lint checks. + +:coverage: + Reproduce the most recent coverage report. + +:clean: + Remove a number of directories and their files: + + - ``.tox`` + + - ``.Python`` + + - ``bin`` + + - ``include`` + + - ``lib`` + + - ``pip-selfcheck`` + + - ``.json`` + +:benchmarks: + Run the applications in the ``benches`` directory to gather performance benchmark data. + + - ``large_resource_set.py`` + + - ``complex_expression.py`` diff --git a/docs/source/index.rst b/docs/source/index.rst index 1e548d7..5e3ab46 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -25,26 +25,33 @@ Pure Python implementation of Google Common Expression Language, https://opensou This implementation has minimal dependencies, runs quickly, and can be embedded into Python-based applications. Specifically, one intent is to be part of Cloud Custodian (C7N) as part of the security policy filter. -Interested in the API? There are three interesting topics: - -- :ref:`integration` -- :ref:`api` -- :ref:`data_structures` - -The integration into another application isn't a trivial ``import``. - .. toctree:: :maxdepth: 2 - :caption: Contents: + :caption: Documentation Content: installation cli configuration integration - api structure + api + development c7n_functions +Integration Overview +==================== + +Interested in the API for using this package? There are three key topics: + +- :ref:`integration` +- :ref:`api` +- :ref:`data_structures` + +The integration into another application is often a bit more than an ``import``. +This is because it involves combining CEL into another DSL. + +The current implementation includes Cloud Custodian (C7N) integration. + Indices and tables ================== diff --git a/docs/source/installation.rst b/docs/source/installation.rst index 42edddc..655b4ca 100644 --- a/docs/source/installation.rst +++ b/docs/source/installation.rst @@ -30,6 +30,7 @@ The optional RE2 package significantly speeds up regular expression matching. python -m pip install cel-python[re2] + .. warning:: In the case where the platform is "darwin" and the architecture is "arm64" and python is "3.13", diff --git a/docs/source/integration.rst b/docs/source/integration.rst index 86ef221..c36bfaa 100644 --- a/docs/source/integration.rst +++ b/docs/source/integration.rst @@ -8,32 +8,26 @@ Application Integration ######################## -We'll look at the essential base case for integration: -evaluate a function given some variable bindings. +We'll look at integration of CEL into another application from four perspectives: -Then we'll look at providing custom function bindings to extend -the environment. +1. We'll look at the essential base case for integration into another application. + This will use an ``Activation`` to provide values for variables. -We'll also look at additional examples from the Go implementation. -This will lead us to providing custom type providers -and custom type adapters. +2. A more sophisticated integration involves extending the environment with custom functions. + This can provide a well-defined interface between CEL expressions and application functionality. -There are a few exception and error-handling cases that are helpful -for writing CEL that tolerates certain kinds of data problems. +3. Some additional examples from the Go implementation show how extend the environment using new types. -Finally, we'll look at how CEL can be integrated into Cloud Custodian. +4. There are a few exception and error-handling cases that are part of working with Python types. -The Essentials -============== +5. Finally, we'll look at how CEL can be integrated into Cloud Custodian (C7N). -Here are two examples of variable bindings - -README ------- +Integration Essentials +====================== -Here's the example taken from the README. +Here's an example of variable bindings taken from a ``README`` example: -The baseline implementation works like this:: +.. code-block:: python >>> import celpy >>> cel_source = """ @@ -46,36 +40,49 @@ The baseline implementation works like this:: >>> ast = env.compile(cel_source) >>> prgm = env.program(ast) - >>> activation = { + >>> context = { ... "account": celpy.json_to_cel({"balance": 500, "overdraftProtection": False}), ... "transaction": celpy.json_to_cel({"withdrawal": 600}) ... } - >>> result = prgm.evaluate(activation) + >>> result = prgm.evaluate(context) >>> result BoolType(False) -The :py:class:`celpy.Environment` can include type adapters and type providers. It's not clear -how these should be implemented in Python or if they're even necessary. +The ``cel_source`` is an expression to be evaluated. +This references variables with names like ``account``, and ``transaction``. + +All CEL evaluation uses an :py:class:`celpy.Environment` object. +The :py:class:`celpy.Environment` is used to provide type annotations for variables. +It can provide a few other properties, including an overall package name, sometimes needed when working with protobuf types. -The compile step creates a syntax tree, which is used to create a final program to evaluate. -Currently, there's a two-step process because we might want to optimize or transform the AST prior -to evaluation. +The :py:meth:`Environment.compile` method creates a abstract syntax tree from the CEL source. +This will be used to create a final program to evaluate. +This method can raise the :py:exc:`CELSyntaxError` exception. -The activation provides specific variable values used to evaluate the program. +The :py:meth:`Environment.program` method creates a runner out of an abstract syntax tree. -To an extent, the Python classes are loosely based on the object model in https://github.com/google/cel-go. +Compiling and building a program is a two-step process to permit optimization or some other transformation the AST prior to evaluation. +The Lark parser (https://lark-parser.readthedocs.io/en/latest/classes.html) is used, and transformers are a first-class feature of this parser. + +The ``context`` mapping defines variables and provides their values. +This is used to evaluate the resulting program object. +The program will produce a value defined in the :py:mod:`celpy.celtypes` module. +In this example, it's a :py:mod:`celpy.celtypes.BoolType` value. + +The CEL types are all specializations of the obvious Python base types. +To an extent, these Python classes are partially based on the object model in https://github.com/google/cel-go. We don't need all the Go formalisms, however, and rely on Pythonic variants. -Simple example using builtin operators +Simple example using builtin types --------------------------------------- Here's an example taken from -https://github.com/google/cel-go/blob/master/examples/README.md +https://github.com/google/cel-go/blob/master/examples/README.md. +This will evaluate the CEL expression ``"Hello world! I'm " + name + "."`` with ``"CEL"`` passed as the ``name`` variable. -Evaluate expression ``"Hello world! I'm " + name + "."`` with ``CEL`` passed as -the ``name`` variable. +This is the original Go code: -.. code:: go +.. code-block:: go import ( "github.com/google/cel-go/cel" @@ -97,7 +104,9 @@ the ``name`` variable. fmt.Println(out) // Output:Hello world! I'm CEL. -Here's the Python version:: +Here's the Python version, following a similar outline: + +.. code-block:: python >>> import celpy >>> cel_source = """ @@ -109,59 +118,59 @@ Here's the Python version:: >>> ast = env.compile(cel_source) >>> prgm = env.program(ast) - >>> activation = { + >>> context = { ... "name": "CEL" ... } - >>> result = prgm.evaluate(activation) + >>> result = prgm.evaluate(context) >>> result "Hello world! I'm CEL." -There's a big open concern here: there's no formal type adapter implementation. -Nothing converts from the input value in the activation to the proper underlying -type. This relies on Python's built-in type conversions. +The steps include: -.. todo:: Handle type adapters properly. +1. Create a :py:class:`celpy.Environment` with annotations for any variables. + These kinds of type definitions are atypical for Python, but are part of the definition of the CEL language. -Function Bindings -================= +2. Use :py:meth:`celpy.Environment.compile` to create an AST. -Here are two more examples of binding, taken from -https://github.com/google/cel-go/blob/master/examples/README.md +3. Use :py:meth:`celpy.Environment.program` to build a :py:class:`celpy.Runner` object that will do the final evaluation. This includes the environment and the AST. -Note the complication here comes from the way the Go implementation resolves overloaded functions. -Each CEL overload of a function is described by a ``("name", [args], result)`` structure. -This allows for multiple type-specific overload versions of a generic function. +4. Use :py:meth:`celpy.Runner.evaluate` to evaluate the program with specific values for the defined variables. -The key of ``("name", [args], result)`` maps to a specific ``arg_name_arg()`` or ``name_arg()`` -overloaded implementation for specific argument types. +In the Go world, there's a formal type adapter to convert input values to the objects used by CEL. +For numerous types, a default adapter handles this. -For example, ``("greet", [StringType, StringType], StringType)`` maps to ``string_greet_string()``. +In Python, on the other hand, we define the type conversions as features of the Python versions of the CEL types. +This approach fits better with native Python programming. -This is emphatically not how Python generally works. A more Pythonic approach is to provide -a single, generic, function which examines the arguments and decides what to do. Python doesn't -generally do overloaded name resolution. -There are two choices: +Function Bindings +================= + +There are two function binding examples in +https://github.com/google/cel-go/blob/master/examples/README.md. -1. Build a mapping from ``("name", [args], result)`` to a specific overloaded implementation. - This pulls argument and result type coercion outside the Python function. - It matches the Go implementation, but can be confusing for Python implementers. - This requires exposing a great deal of machinery already available in a Python function - definition. +There is a complication here that based on the way the Go resolves overloaded functions. +In Go, each overload of a function is described by a ``("name", [args], result)`` data structure. +The key of ``("name", [args], result)`` maps to a specific ``arg_name_arg()`` or ``name_arg()`` overloaded implementation for specific argument types. +This allows for multiple type-specific overload versions of a generic function. -2. Ignore the complex type exposture techniques that Go requiees and dispatch to a Python function. - The Python function will sort out type variants and handle argument value coercion on its own. - This simplifies implementation down to name resolution. - Indeed, the type mapping rules can introspect Python's type annotations on the function - definition. +For example, a ``("greet", [StringType, StringType], StringType)`` structure is expected to map to a function ``string_greet_string()`` that has the expected signature. -We follow the 2nd alternative. The Python function binding relies -- exclusively -- on introspection -of the function provided. +This is emphatically not how Python generally works. +We follow a more Pythonic approach is to provide a single, generic, function which examines the arguments and decides what to do. +Outside type-checking, Python doesn't depend on overloaded name resolution. -Custom function on string type ------------------------------- +This means a Python function must then sort out type variants and handle argument value coercion on its own. +For most cases, the ``match/case`` statement is helpful for this. +The :py:func:`functools.singledispatch` decorator can also be helpful for this. -Evaluate expression ``i.greet(you)`` with: +The two examples have slightly different approaches to the CEL expression. +These are important in Go, but less important in Python. + +Custom function in Go +--------------------------------------- + +We want to evaluate the CEL expression ``i.greet(you)`` with: .. parsed-literal:: @@ -169,14 +178,14 @@ Evaluate expression ``i.greet(you)`` with: you -> world greet -> "Hello %s! Nice to meet you, I'm %s." +The idea here is the new ``greet()`` behaves like a method of a String. +The actual implementation, however, is not a method; it's a function of two arguments. -First we need to declare two string variables and `greet` function. -`NewInstanceOverload` must be used if we want to declare function which will -operate on a type. First element of slice passed as `argTypes` into -`NewInstanceOverload` is declaration of instance type. Next elements are -parameters of function. +First we need to declare two string variables and a ``greet()`` function. +In Go, a ``NewInstanceOverload`` must be used to provide annotations for variables and the function. +Here's the Go implementation: -.. code:: go +.. code-block:: go decls.NewVar("i", decls.String), decls.NewVar("you", decls.String), @@ -186,12 +195,15 @@ parameters of function. decls.String)) ... // Create env and compile +We've omitted the Go details of creating an environment and compiling the CEL expression. +These aren't different from the previous examples. -Let's implement `greet` function and pass it to `program`. We will be using -`Binary`, because `greet` function uses 2 parameters (1st instance, 2nd -function parameter). +Separately, a ``greetFunc()`` function must be defined. +In Go, this function is then bound to the ``"string_greet_string"`` overload, +ready for evaluation. +Here's the Go implementation: -.. code:: go +.. code-block:: go greetFunc := &functions.Overload{ Operator: "string_greet_string", @@ -208,7 +220,22 @@ function parameter). fmt.Println(out) // Output:Hello world! Nice to meet you, I'm CEL. -Here's the Python version:: +What's essential is defining some type information, then defining variables and functions that fit those types. + +The Python version has the same outline: + +1. An :py:class:`celpy.Environment` with type annotations for the two variables and the function. + +2. Compile the source. + +3. Define the ``greet()`` function. While the CEL syntax of ``i.greet(you)`` looks like a method +of the ``i`` variable's class, the function is simply has two positional parameters. + +4. Provide function implementation when creating the final :py:class:`celpy.Runner` instance. + +5. Evaluate the program with specific values for the two variables. + +.. code-block:: python >>> import celpy >>> cel_source = """ @@ -224,53 +251,36 @@ Here's the Python version:: >>> def greet(lhs: celpy.celtypes.StringType, rhs: celpy.celtypes.StringType) -> celpy.celtypes.StringType: ... return "Hello {1:s}! Nice to meet you, I'm {0:s}.\\n".format(lhs, rhs) >>> prgm = env.program(ast, functions=[greet]) - >>> activation = { + >>> context = { ... "i": "CEL", "you": "world" ... } - >>> result = prgm.evaluate(activation) + >>> result = prgm.evaluate(context) >>> result "Hello world! Nice to meet you, I'm CEL.\\n" -Define custom global function ------------------------------ - -Evaluate expression ``shake_hands(i,you)`` with: +The key concept here is to distinguish between three distinct attributes: -.. parsed-literal:: +1. Type annotations associated with variables or functions. - i -> CEL - you -> world - shake_hands -> "%s and %s are shaking hands." +2. The function implementations used to build the :py:class:`celpy.Runner`. + The method-like syntax of ``i.greet(you)`` is evaluated as ``greet(i, you)``. +3. The variable values, which provide a context in which the runner evaluates the CEL expression. -In order to declare global function we need to use `NewOverload`: +This reflects the idea that one CEL expression may be used to process data over and over again. -.. code:: go +Define custom global function +----------------------------- - decls.NewVar("i", decls.String), - decls.NewVar("you", decls.String), - decls.NewFunction("shake_hands", - decls.NewOverload("shake_hands_string_string", - []*exprpb.Type{decls.String, decls.String}, - decls.String)) - ... // Create env and compile. +In Go, this is a small, but important different.ce +We want to evaluate the expression ``shake_hands(i,you)``. +This uses a global function syntax instead of method syntax. - shakeFunc := &functions.Overload{ - Operator: "shake_hands_string_string", - Binary: func(lhs ref.Val, rhs ref.Val) ref.Val { - return types.String( - fmt.Sprintf("%s and %s are shaking hands.\n", lhs, rhs)) - }} - prg, err := env.Program(c, cel.Functions(shakeFunc)) +While Go has slight differences in how the function is defined, in Python, there is no change. - out, _, err := prg.Eval(map[string]interface{}{ - "i": "CEL", - "you": "world", - }) - fmt.Println(out) - // Output:CEL and world are shaking hands. +Here's the Python version: -Here's the Python version:: +.. code-block:: python >>> import celpy >>> cel_source = """ @@ -286,24 +296,25 @@ Here's the Python version:: >>> def shake_hands(lhs: celpy.celtypes.StringType, rhs: celpy.celtypes.StringType) -> celpy.celtypes.StringType: ... return f"{lhs} and {rhs} are shaking hands.\\n" >>> prgm = env.program(ast, functions=[shake_hands]) - >>> activation = { + >>> context = { ... "i": "CEL", "you": "world" ... } - >>> result = prgm.evaluate(activation) + >>> result = prgm.evaluate(context) >>> result 'CEL and world are shaking hands.\\n' +The ``shake_hands()`` function is essentially the same as the ``greet()`` function in the previous example. -For more examples of how to use CEL, see +For more examples of how to use CEL from Go, see https://github.com/google/cel-go/tree/master/cel/cel_test.go -Examples from Go implementation -================================ +More Examples from Go implementation +===================================== -See https://github.com/google/cel-go/blob/master/README.md +See https://github.com/google/cel-go/blob/master/README.md for five more examples. -.. code:: +.. code-block:: // Check whether a resource name starts with a group name. resource.name.startsWith("/groups/" + auth.claims.group) @@ -321,9 +332,10 @@ See https://github.com/google/cel-go/blob/master/README.md // in the JSON case. has(message.field) -Following one of the more complete examples through the README +Here's the first example, ``resource.name.startsWith("/groups/" + auth.claims.group)``. +The Go code is as follows: -.. code:: go +.. code-block:: go import( "github.com/google/cel-go/cel" @@ -353,7 +365,10 @@ Following one of the more complete examples through the README "group": "acme.co"}) fmt.Println(out) // 'true' -This has the following Python implementation:: +This has a Python implementation which is substantially similar. +Here's the Python code: + +.. code-block:: python >>> import celpy >>> decls = { @@ -363,60 +378,50 @@ This has the following Python implementation:: >>> env = celpy.Environment(annotations=decls) >>> ast = env.compile('name.startsWith("/groups/" + group)') >>> prgm = env.program(ast) - >>> activation = { + >>> context = { ... "name": "/groups/acme.co/documents/secret-stuff", ... "group": "acme.co", ... } - >>> result = prgm.evaluate(activation) + >>> result = prgm.evaluate(context) >>> result BoolType(True) +The general outline of compile, create a :py:class:`celpy.Runner`, and use :py:meth:`celpy.Runner.evaluate` to evaluate the CEL expression in a specific context is the central point here. + Exceptions and Errors ====================== -Exceptions raised in Python world will (eventually) crash the CEL evluation. -This gives the author of an extension function the complete traceback to help -fix the Python code. +Exceptions raised in Python world will (eventually) crash the CEL evaluation. +This gives the author of an extension function the complete traceback to help fix the Python code. No masking or rewriting of Python exceptions ever occurs in extension functions. -A special :exc:`celpy.EvalError` exception can be used in an extension function -to permit CEL's short-circuit logic processing to silence this exception. See the -https://github.com/google/cel-go/blob/master/README.md#partial-state for more examples -of how the short-circuit (partial state) operations work. +A special :py:exc:`celpy.CELEvalError` exception can be used in an extension function to permit CEL's short-circuit logic processing to check and ignore an exception. +See the https://github.com/google/cel-go/blob/master/README.md#partial-state for more examples of how the short-circuit (partial state) operations work. -An extension function must **return** a :exc:`celpy.EvalError` object -to allow processing to continue in spite of an uncomputable value. +An extension function can **return** a :py:exc:`celpy.CELEvalError` object instead of raising it. +This can allow processing to continue in spite of an uncomputable value. -:: +.. code-block:: python from celpy import * def my_extension(a: Value) -> Value: try: return celtypes.UintType(64 // a) except DivideByZeroError as ex: - return EvalError(f"my_extnsion({a}) error") - -The returned exception object allows short-circuit processing. For example, - -:: - - false && my_extension(0) + return CELEvalError(f"my_extension({a}) error") -This evaluates to ``false``. If computed, any :exc:`celpy.EvalError` object will be silently ignored. +The returned exception object allows short-circuit processing. +For example, the CEL expression ``false && my_extension(0)`` evaluates to ``false``. +If computed, any :exc:`celpy.CELEvalError` objects will be silently ignored because the short-circuit result is known from the presence of a ``false`` value. -On the other hand, - -:: - - true && my_extension(0) - -This will result in a visible :exc:`celpy.EvalError` result from the extension function. +On the other hand, the CEL expression ``true && my_extension(0)`` results in the :exc:`celpy.CELEvalError` result from the extension function. This will eventually be raised as an exception, so the framework using ``celpy`` can track this run-time error. -Cloud Custodian -=============== +Cloud Custodian (C7N) Integration +================================== Custodian Filters can be evaluated by CEL. +The idea is to extend the YAML-based DSL for policy documents to introduce easier-to-read expressions. As noted in https://github.com/cloud-custodian/cloud-custodian/issues/5759, a filter might look like the following:: @@ -432,15 +437,13 @@ following:: This replaces a complex sequence of nested ``- and:`` and ``- or:`` sub-documents with a CEL expression. -C7N processes resources by gathering resources, creating an instance of a subclass of the ``Filter`` -class, and evaluating an expression like ``take_action = list(filter(filter_instance, resource_list))``. +C7N processioning works by gathering resources, creating an instance of a subclass of the ``Filter`` class, and evaluating an expression like ``take_action = list(filter(filter_instance, resource_list))``. -The C7N filter expression in a given policy document is componsed of one or more atomic filter clauses, -combined by ``and``, ``or``, and ``not`` operators. +The C7N filter expression in a given policy document is composed of one or more atomic filter clauses, combined by ``and``, ``or``, and ``not`` operators. The filter as a whole is handled by the ``__call__()`` methods of subclasses of the ``BooleanGroupFilter`` class. Central to making this work is making the CEL expression into a function that can be applied to the ``resource`` object. -It appears that all CEL operations will need to have a number of values in their activations: +All CEL versions of a filter will need to have a the following two values in their activations: :resource: A :py:class:`celtypes.MapType` document with the resource details. @@ -448,14 +451,15 @@ It appears that all CEL operations will need to have a number of values in their :now: A :py:class:`celtypes.TimestampType` object with the current time. -Additional "global" objects may also be helpful. Baseline C7N Example -------------------- -The essence of the integration is to provide a resource to a function and receive a boolean result. +The essence of the integration is to provide a resource description to a function defined as a CEL expression, and receive a boolean result. + +Here's a base example: -Here's a base example:: +.. code-block:: python >>> import celpy >>> env = celpy.Environment() @@ -484,22 +488,26 @@ Here's a base example:: >>> prgm.evaluate(activation) BoolType(True) +In this case, the context contained only one variable, ``resource``. +It didn't require a definition of ``now``. + Bulk Filter Example ------------------- Pragmatically, C7N works via code somewhat like the following: -:: +.. code-block:: resources = [provider.describe(r) for r in provider.list(resource_type)] map(action, list(filter(cel_program, resources))) -An action is applied to those resources that pass some filter test. The filter looks for items not compliant -with policies. +An action is applied to those resources that pass some filter test. +Often, the action disables a resource to prevent data compromise. +The filter looks for items not compliant with policies so they can be deleted or disabled. The ``cel_program`` in the above example is an executable CEL program wrapped into a C7N ``Filter`` subclass. -:: +.. code-block:: >>> import celpy >>> import datetime @@ -537,23 +545,25 @@ The ``cel_program`` in the above example is an executable CEL program wrapped in >>> actionable [{'name': 'bad1', 'tags': {'not-owner': 'oops'}}, {'name': 'bad2', 'tags': {'owner': None}}] +For each resource, the ``tag_policy_filter`` object applied an internal ``self.prgm`` to the resource. +The internal ``self.prgm`` was built from the policy expression, stated in CEL. C7N Filter and Resource Types ------------------------------- -There are several parts to handling the various kinds of C7N filters in use. +The :py:mod:`celpy.c7nlib` module provides filter subclasses that include CEL processing. +There are two kinds of C7N filters in use. -1. The :py:mod:`c7n.filters` package defines about 23 generic filter classes, all of which need to - provide the ``resource`` object in the activation, and possibly provide a library of generic - CEL functions used for evaluation. - The general cases are of this is handled by the resource definition classes creating values in a JSON document. +1. The :py:mod:`c7n.filters` package defines about 23 generic filter classes. + These apply to a ``resource`` object. + Additionally, there's a library of generic functions used for evaluation. + Generally, the resource definition classes create values in a JSON document. These values reflect the state of the resource and any closely-related resources. 2. The :py:mod:`c7n.resources` package defines a number of additional resource-specific filters. - All of these, similarly, need to provide CEL values as part of the resource object. - These classes can also provide additional resource-specific CEL functions used for evaluation. + These classes can also provide additional resource-specific processing. -The atomic filter clauses have two general forms: +The atomic filter clauses within a policy document have two general forms: - Those with "op". These expose a resource attribute value, a filter comparison value, and an operator. @@ -566,8 +576,8 @@ The atomic filter clauses have two general forms: The breakdown of ``filter`` rules in the C7N policy schema has the following counts. .. csv-table:: - :header: category, count, notes + "('Common', 'Op')",21,"Used for more than one resource type, exposes resource details to CEL" "('Common', 'No-Op')",15,"Used for more than one resource type, does not expose resource details" "('Singleton', 'Op')",27,"Used for exactly one resource type, exposes resource details to CEL" diff --git a/docs/source/structure.rst b/docs/source/structure.rst index b2ee74c..5517636 100644 --- a/docs/source/structure.rst +++ b/docs/source/structure.rst @@ -4,51 +4,365 @@ .. _`data_structures`: -############### -Data Structures -############### +################################ +Architecture and Design +################################ -Run-Time -======== +We'll start with the C4 views: -An external client depends on the :py:class:`celpy.Environment`. +- `Context` -The :py:class:`celpy.Environment` builds the initial AST and the final runnable "program." -The :py:class:`celpy.Environment` may also contain a type provider and type adapters. +- `Container` -- this isn't too interesting, but it can help to see this. -The :py:class:`celpy.Environment` also builds -an :py:class:`celpy.evaluation.Activation` with the variable and function bindings -and the default package. +- `Components` -The :py:class:`celpy.evaluation.Activation` create a kind of chainmap for name -resolution. The chain has the following structure: + This is a collection of various design notes describing some implementation details. -- The end of the chain is the built-in defaults. + - `Compile-Time`_ -- A layer on top of this can be provided as part of integration into some other app or framework. + - `Evaluation-Time`_ -- The next layer is the "current" activation when evaluating a given expression. - This often has command-line variables. + - `CEL Types`_ + + - `Transpiler Missing Names`_ + + - `The member-dot Production`_ + +The code view is in the :ref:`api` section. + +Context +======= + +There are two distinct contexts for CEL Python: + +- The CLI -- as a stand-alone application. + +- As an importable module to provide expressions to a DSL. + +.. uml:: + + @startuml + skinparam actorStyle awesome + left to right direction + + package celpy { + package library { + usecase lib1 as "extend DSL with expressions" + usecase lib2 as "create program" + usecase lib3 as "evaluate program in context" + lib1 --> lib2 + lib1 --> lib3 + } + + package cli { + usecase cli1 as "**expr** features + --- + Use the -n option" + usecase cli2 as "**test** features + --- + Use the -nb options" + usecase cli3 as "**jq** features + Newline-Delimited or single JSON doc" + usecase cli4 as "interactive computation + --- + use the -i option" + } + + } + + actor app as "some app with a DSL" + app --> lib1 + + actor bash as "shell script" + bash --> cli1 + bash --> cli2 + bash --> cli3 + + actor user + user --> cli4 + + app <|- [c7n] + @enduml + +From the CLI, the ``celpy`` application has a number of use cases: + +- A shell script can use ``celpy`` as a command to replace other shell commands, including **expr**, **test**, and **jq**. + +- A person can run ``celpy`` interactively. + This allows experimentation. + It also supports exploring very complex JSON documents to understand their structure. + +As a library, an application (for example, C7N) can import ``celpy`` to provide an expression feature for the DSL. +This provides well-defined semantics, and widely-used syntax for the expression language. +There's an explicit separation between building a program and executing the program to allow caching an expression for multiple executions without the overhead of building a Lark parser or compiling the expression. + +Container +========= + +As a CLI, this is part of a shell script. It runs where the script runs. + +As a library, this is improted into the application to extend the DSL. + +There are no services offered or used. + +Components +========== + +The Python code base has a number of modules. + +- ``__init__`` -- the ``celpy`` package as a whole. + +- ``__main__`` -- the main applications used when running ``celpy``. + +- ``celparser`` -- a **Facade** for the Lark parser. + +- ``evaluation`` -- a **Facade** for run-time evaluation. + +- ``celtypes`` -- the underlying Python implementations of CEL data structures. + +- ``c7nlib``-- a collection of components the C7N can use to introduce CEL filters. + +- ``adapter`` -- Some JSON serialization components. + +Here's the conceptual organiation + +.. uml:: + + @startuml + + package celpy { + component "~__init__" as init + component "~__main__" as main + component adapter + component c7nlib + component celparser + component celtypes + component evaluation + component cel.lark + } + init --> celtypes + init --> adapter + init --> celparser + init--> evaluation + + main --> init + main --> celparser + main --> adapter + main --> evaluation + + adapter --> celtypes + + c7nlib --> evaluation + c7nlib --> adapter + c7nlib --> celtypes + c7nlib --> init + + celparser --> cel.lark + celparser --> lark + + evaluation --> lark + evaluation --> celtypes + + package lark { + } + @enduml + +While there is a tangle of dependencies, there are three top-level "entry points" for ``celpy``. + +- The ``__main__`` module is the CLI application. + +- The ``c7nlib`` module exposes CEL functionality in a form usable by Cloud Custodian filter definitions. + This library provides useful components to perform Custodian-related computations. + +- The ``__init__`` module is exposes the most useful parts of ``celpy`` for integration woth another application. + +Compile-Time +------------- + +Here are the essential classes used to compile a CEL expression and prepare it for evaluation. + +.. uml:: + + @startuml + hide empty members + + class Environment { + package: str + annotations: dict[str, Annotation] + compile(text: str) -> lark.Tree + program(expr: lark.Tree, functions: dict) -> Runner + } + + class celparser.CELParser{ + parse(text: str) + } + Environment *-- CELParser + + class lark.Tree {} + CELParser --> lark.Tree : "Creates" + + abstract class Runner { + ast: Tree + evaluate(context: Context) -> Value + } + Environment --> Runner : "Creates" + Runner o-- lark.Tree + Runner o-- "0..m" CELFunction + + class InterpretedRunner + Runner <|-- InterpretedRunner + + class evaluation.Evaluator + InterpretedRunner *-- Evaluator + + class CompiledRunner + Runner <|-- CompiledRunner + + class evaluation.Transpiler + CompiledRunner *-- Transpiler + + class evaluation.Context << (T,orchid) Type>> { + key: str + value: Result | NameContainer + } + Runner o--- "0..m" Context + + class CELFunction <> + + class Annotation << (T,orchid) Type>> + Environment o-- "0..m" Annotation + + class TypeType + Annotation <|-- TypeType + Annotation <|-- CELFunction -- A transient top-most layer is used to create a local variable binding - for the macro evaluations. + @enduml -The AST is created by Lark from the CEL expression. +The fundamental sequence of operations is +1. Create an :py:class:`celpy.Environment` with any needed :py:class:`celpy.Annotation` instances. + For the most part, these are based on the overall application domain. + Any type definitions should be subclasses of :py:class:`celpy.TypeType` or a callable function defined by the :py:class:`celpy.CELFunction` type. + +2. Use the :py:class:`celpy.Environment` to compile the CEL text to create a parse tree. + +3. Use the :py:class:`celpy.Environment` to create a :py:class:`celpy.Runner` instance from the parse tree and any function definitions that override or extend the predefined CEL environment. + +4. Evaluate the :py:class:`celpy.Runner` with a :py:class:`celpy.Context`. + The :py:class:`celpy.Context` provides specific values for variables required for evaluation. + Generally, each variable should have an :py:class:`celpy.Annotation` defined in the :py:class:`celpy.Environment`. + +The :py:class:`celpy.Runner` can be evaluated with any number of distinct :py:class:`celpy.Context` values. +This amortizes the cost of compilation over multiple executions. + +Evaluation-Time +---------------- + +Here's the classes to evaluate a CEL expression. + +.. uml:: + + @startuml + hide empty members + + abstract class Runner { + ast: Tree + evaluate(context: Context) -> Value + } + Environment --- Runner : "Created By <" + Runner o-- "0..m" CELFunction + Runner o-- Context + + class lark.Tree + Tree --* Runner + + class InterpretedRunner <> + Runner <|-- InterpretedRunner + + abstract class lark.Interpreter + + class evaluation.Evaluator { + activation: Activation + functions: dict[str, CELFunction] + evaluate() -> Value + } + lark.Interpreter <|--- Evaluator + InterpretedRunner *-- Evaluator + + class CompiledRunner <> + Runner <|-- CompiledRunner + + InterpretedRunner -[hidden]> CompiledRunner + + class evaluation.Transpiler { + functions: dict[str, CELFunction] + transpile() + evaluate() -> Value + } + CompiledRunner *-- Transpiler + lark.Interpreter <|--- Transpiler + + class evaluation.Activation { + annotations: Annotation + identifiers: dict[str, Result | CELFunction] + } + Runner *-- Activation : "Uses" + Runner --> Activation : "Creates" + Activation --> Activation : "based on" + + class Annotation << (T,orchid) Type>> + Runner *-- "0..m" Annotation + Annotation --o Activation : Initializes + CELFunction --o Activation : Initializes + Context --o Activation : Initializes + + @enduml + +The evalation of the CEL expression is done via a :py:class:`celpy.Runner` object. There are two :py:class:`celpy.Runner` implementations. -- The :py:class:`celpy.InterpretedRunner` walks the AST, creating the final result or exception. +- The :py:class:`celpy.InterpretedRunner` walks the AST, creating the final result :py:class:`celpy.Value` or :py:class:`celpy.CELEvalError` exception. + This uses a :py:class:`celpy.evaluation.Activation` to perform the evaluation. + +- The :py:class:`celpy.CompiledRunner` transpiles the AST into a Python sequence of statements. + The internal :py:func:`compile` creates a code object that can then be evaluated with a given :py:class:`celpy.evaluation.Activation` + The internal :py:func:`exec` functions performs the evaluation. + +The subclasses of :py:class:`celpy.Runner` are **Adapter** classes to provide a tidy interface to the somewhat more complex :py:class:`celpy.Evaluator` or :py:class:`celpy.Transpiler` objects. +In the case of the :py:class:`celpy.InterpretedRunner`, evaluation involves creating an :py:class:`celpy.evaluation.Activation` and visiting the AST. +Whereas, the :py:class:`celpy.CompiledRunner` must first visit the AST to create code. At evaluation time, it create an :py:class:`celpy.evaluation.Activation` and uses :py:func:`exec` to compute the final value. + +The :py:class:`celpy.evaluation.Activation` contains several things: + +- The :py:class:`Annotation` definitions to provide type information for identifiers. -- The :py:class:`celpy.CompiledRunner` transforms the AST to remove empty rules. Then emits - the result as a Python expression. It uses the Python internal :py:func:`compile` and :py:func:`eval` functions - to evaluate the expression. +- The :py:class:`CELFunction` functions that extend or override the built-in functions. + +- The values for identifiers. + +The :py:class:`celpy.evaluation.Activation` is a kind of chainmap for name resolution. +The chain has the following structure: + +- The end of the chain has the built-in defaults. + (This is the bottom-most base definition.) + +- A layer on top of this can offer types and functions which are provided to integrate into the containing app or framework. + +- The next layer is the "current" activation when evaluating a given expression. + For the CLI, this has the command-line variables. + For other integrations, these are the input values. + +- A transient layer on top of this is used to create a local variable binding for the macro evaluations. + These can be nested, and introduce the macro variable as a temporary annotation and value binding. CEL Types -========== +---------- There are ten extension types that wrap Python built-in types to provide the unique CEL semantics. +- :py:class:`celtypes.TypeType` is a supertype for CEL types. + - :py:class:`celtypes.BoolType` wraps ``int`` and creates additional type overload exceptions. - :py:class:`celtypes.BytesType` wraps ``bytes`` it handles conversion from :py:class:`celtypes.StringType`. @@ -73,4 +387,83 @@ There are ten extension types that wrap Python built-in types to provide the uni from ``datetime.timedelta``, ``int``, and ``str`` values. Additionally, a :py:class:`celtypes.NullType` is defined, but does not seem to be needed. It hasn't been deleted, yet. -but should be considered deprecated. +It should be considered deprecated. + +Transpiler Missing Names +==================================================== + +The ``member_dot`` transpilation with a missing name will be found at evaluation time via ``member.get('IDENT')``. This raises No Such Member in Mapping error. + +The ``primary :: ident`` evaluation can result in one of the following conditions: + + - ``ident`` denotes a type definition. The value's type is ``TypeType``. + The value is a type reference ``bool`` becomes ``celpy.celtypes.BoolType``. + + - ``ident`` denotes a built-in function. The value's type is ``CELFunction``. + The value is the Python function reference. + + - ``ident`` denotes an annotation, but the value's type is neither ``TypeType`` nor ``CELFunction``. + + The transpiled value is ``f"activation.{ident}"``, assuming it will be a defined variable. + + If, at ``exec()`` time the name is not in the Activation with a value, a ``NameError`` exception will be raised that becomes a ``CELEvalError`` exception. + + +The Member-Dot Production +========================= + +Consider ``protobuf_message{field: 42}.field``. +This is parsed using the following productions. + +.. code-block:: bnf + + member : member_dot | member_dot_arg | member_item | member_object | primary + member_dot : member "." IDENT + member_object : member "{" [fieldinits] "}" + +The ``member_object`` will be a ``primary`` which can be an ``ident``. +It MUST refer to the Annotation (not the value) because it has ``fieldinits``. +All other choices are (generally) values. +They can be annotations, which means ``bool.type()`` works the same as ``type(bool)``. + +Here's ``primary`` production, which defines the ``ident`` in the ``member`` production. + +.. code-block:: bnf + + primary : dot_ident_arg | dot_ident | ident_arg | ident + | paren_expr | list_lit | map_lit | literal + +The ``ident`` is not **always** transpiled as ``activation.{name}``. +Inside ``member_object``, it's ``activation.resolve_name({name})``. +Outside ``member_object``, it can be ``activation.{name}`` because it's a simple variable. + +It may make sense to rename the :py:meth:`Activation.resolve_name` method to :py:meth:`Activation.get`. + +This, however, overloads the ``get()`` method. +This has type hint consequences. + +.. important:: + + The ``member`` can be any of a variety of objects: + + - ``NameContainer(Dict[str, Referent])`` + + - ``Activation`` + + - ``MapType(Dict[Value, Value])`` + + - ``MessageType(MapType)`` + + All of these classes must define a ``get()`` method. + The nuance is the ``NameContainer`` is also a Python ``dict`` and there's an + overload issue between that ``get()`` and other ``get()`` definitions. + +The Transpilation **currently** leverages a common method named ``get()`` for all of these types. +This is a Pythonic approach, but, the overload for the ``NameContainer`` (a ``Dict`` subclass) isn't quite right: +it doesn't return a ``Referent``, but the value from a ``Referent``. + +A slightly smarter approach is to define a ``get_value(member, 'name')`` function that uses a match/case structure to do the right thing for each type. The problem is, the result is a union of type, value, function, and any of these four containers! + +Another possibility is to leverage the Annotations. +They **can** provide needed type information to discern which method with specific result type. + diff --git a/features/README.rst b/features/README.rst index 7b75e62..90ee2e8 100644 --- a/features/README.rst +++ b/features/README.rst @@ -6,6 +6,7 @@ We start with https://github.com/google/cel-spec/tree/master/tests/simple/testda as the acceptance test suite. These files are captured as of commit 7e251cc. +This is from Nov 18, 2020, version 0.5.0. We parse the Text serialization of protobuf files to create Gherkin test specifications @@ -28,129 +29,8 @@ To run a subset, pick a feature file. PYTHONPATH=src behave features/basic.feature -We don't use a lot of tags because the Gherkin is derived from source textproto. +Building the Conformance Test Features +====================================== - -Building the Gherkin -==================== - -The conformance tests are in the https://github.com/google/cel-spec.git repository. -They are all protobuf objects, serialized into ``textproto``. - -We translate these into Gherkin. -See ``tools/textproto_to_gherkin.py`` for the application. - -Here's the textproto:: - - test { - name: "self_eval_int_zero" - expr: "0" - value: { int64_value: 0 } - } - -Here's the Gherkin:: - - Scenario: "self_eval_int_zero" - Given disable_check parameter is None - And type_env parameter is None - And bindings parameter is None - When CEL expression "0" is evaluated - Then value is IntType(0) - And eval_error is None - -We've made a bunch of test features explicit. - -The Gherkin creation is controlled by a Makefile that does several things. - -- ``make all`` checks the cel-spec repository for ``.textproto`` files, - moves them to the local ``features`` directory and converts them to ``.features`` files. - -- ``make scan`` runs the ``textproto_to_gherkin.py`` app to check syntax on the protobuf - files. This is used as a smoke test when adjusting the text protobuf grammar. - -- ``make clean-broken`` cleans up the empty ``.feature`` files. These are a consequence of - ``.textproto`` files that cannot be parsed. - -- ``make clean`` removes the ``.feature`` and ``.textproto`` files. - -A good way to use this is to do a checkout from https://github.com/google/cel-spec.git into -an adjacent directory. By deafult, the Makefile looks for ``../../cel-spec``. If this doesn't work -for you, set the ``CEL_SPEC_PATH`` environment variable. - -See ``tools/test_textproto.lark`` for the grammar used to parse the text protobuf. -We aren't interested in a full protobuf implemetation, but enough of an implementation -to parse the tests. - -See https://github.com/protocolbuffers/protobuf/blob/master/python/google/protobuf/text_format.py - -On Gherkinization -================= - -The question arises on how best to serialize descriptions of the objects created by a CEL evaluation. -We have three notations to deal with. - -- Source Protobuf text. - -- Final Python object used by Behave. - -- Some intermediate text representation in the Gherkin. - -The idea is to write Gherkin tests that match the protobuf source closely, but can be processed by -Behave without too much overhead. (There are over 900 conformance tests; we don't want to take all day.) - -There are three broad choices for Gherkin representation of expected results. - -- Protobuf, unmodified. ``{ int64_value: -1 }``. - This pushes the final parsing into the conformance test step definitions. - -- Text representation of the Python target object. ``celpy.celtypes.IntType(-1)``. - This is highly Python-specific and not of general use to other implementers. - -- An intermediate representation. ``Value(value_type='int64_value', source='x')``. - This preserves the protobuf semantics in a language-neutral form. - This can be parsed into Python, or used by other languages. - -The Python representation leverages the following class definition - -:: - - class Value(NamedTuple): - """ - From a protobuf source ``{ int64_value: -1 }``, create an object ``celpy.celtypes.IntType(-1)``. - """ - value_type: str - source: str - - @property - def value(self) -> Any: - ... - - -This permits us to describe a simple object in Gherkin. It preserves the original protobuf detail, -but in a form that can be deserialized more easily by the Python ``eval()`` function. - -A protobuf object like ``{ int64_value: 0 }`` could be modeled as a simple ``0`` in the Gherkin. -For a specific subset of available types the type mapping is not a problem. -For protobuf objects, however, the details matter. - -Here are protobuf objects used in the test cases, their Gherkin representation, and the ``celtypes`` class. - -.. csv-table:: - - "{ int64_value: x }","Value(value_type='int64_value', source='x')",IntType(x) - "{ uint64_value: x }","Value(value_type='uint64_value', source='x')",UintType(x) - "{ double_value: x }","Value(value_type='double_value', source='x')",DoubleType(x) - "{ null_value: NULL_VALUE }","Value(value_type='null_value', source='NULL_VALUE')",None - "{ bool_value: x }","Value(value_type='bool_value', source='x')",BoolType(x) - "{ string_value: ""x"" }","Value(value_type='string_value', source='""x""')",str(x) - "{ bytes_value: ""x"" }","Value(value_type='bytes_value', source='""x""')",bytes(x) - "{ number_value: {'value': 'x'} }","ObjectValue(source='...')",DoubleType(x) - -The ``celtypes`` classes are all subclasses of Python built-in types. - -The protobuf mappings are more complex. - -Building the Protobuf Definitions -================================= - -Build the ``TestAllTypes`` protobuf for use by the Dynamic tests that create protobuf objects +The ``tools`` directory has a ``pb2g.py`` application that builds the test suite. +See the ``tools/README.rst`` for more information. diff --git a/features/basic.feature b/features/basic.feature index 09e3511..d3f40ac 100644 --- a/features/basic.feature +++ b/features/basic.feature @@ -230,7 +230,8 @@ Scenario: self_eval_ascii_escape_seq Scenario: self_eval_bound_lookup # type:{primitive:INT64} - Given type_env parameter "x" is TypeType(value='INT64') + # Given type_env parameter "x" is TypeType(value='INT64') + Given type_env parameter "x" is INT64 # int64_value:123 Given bindings parameter "x" is IntType(source=123) @@ -284,7 +285,8 @@ Scenario: unbound_is_runtime_error Scenario: false # type:{primitive:BOOL} - Given type_env parameter "false" is TypeType(value='BOOL') + # Given type_env parameter "false" is TypeType(value='BOOL') + Given type_env parameter "false" is BOOL # bool_value:true Given bindings parameter "false" is BoolType(source=True) @@ -297,7 +299,8 @@ Scenario: false Scenario: true # type:{primitive:BOOL} - Given type_env parameter "true" is TypeType(value='BOOL') + # Given type_env parameter "true" is TypeType(value='BOOL') + Given type_env parameter "true" is BOOL # bool_value:false Given bindings parameter "true" is BoolType(source=False) @@ -310,7 +313,8 @@ Scenario: true Scenario: null # type:{primitive:BOOL} - Given type_env parameter "null" is TypeType(value='BOOL') + # Given type_env parameter "null" is TypeType(value='BOOL') + Given type_env parameter "null" is BOOL # bool_value:true Given bindings parameter "null" is BoolType(source=True) diff --git a/features/comparisons.feature b/features/comparisons.feature index 0740e8d..cda2c4a 100644 --- a/features/comparisons.feature +++ b/features/comparisons.feature @@ -1214,7 +1214,8 @@ Scenario: key_in_mixed_key_type_map_error Scenario: bytes_gt_left_false # type:{primitive:BYTES} - Given type_env parameter "x" is TypeType(value='BYTES') + # Given type_env parameter "x" is TypeType(value='BYTES') + Given type_env parameter "x" is BYTES # bytes_value:"\x00" Given bindings parameter "x" is BytesType(source=b'\x00') @@ -1227,7 +1228,8 @@ Scenario: bytes_gt_left_false Scenario: int_lte_right_true # type:{primitive:INT64} - Given type_env parameter "x" is TypeType(value='INT64') + # Given type_env parameter "x" is TypeType(value='INT64') + Given type_env parameter "x" is INT64 # int64_value:124 Given bindings parameter "x" is IntType(source=124) @@ -1240,7 +1242,8 @@ Scenario: int_lte_right_true Scenario: bool_lt_right_true # type:{primitive:BOOL} - Given type_env parameter "x" is TypeType(value='BOOL') + # Given type_env parameter "x" is TypeType(value='BOOL') + Given type_env parameter "x" is BOOL # bool_value:true Given bindings parameter "x" is BoolType(source=True) @@ -1253,7 +1256,8 @@ Scenario: bool_lt_right_true Scenario: double_ne_left_false # type:{primitive:DOUBLE} - Given type_env parameter "x" is TypeType(value='DOUBLE') + # Given type_env parameter "x" is TypeType(value='DOUBLE') + Given type_env parameter "x" is DOUBLE # double_value:9.8 Given bindings parameter "x" is DoubleType(source=9.8) @@ -1266,7 +1270,8 @@ Scenario: double_ne_left_false Scenario: map_ne_right_false # type:{map_type:{key_type:{primitive:STRING} value_type:{primitive:STRING}}} - Given type_env parameter "x" is TypeType(value='map_type') + # Given type_env parameter "x" is TypeType(value='map_type') + Given type_env parameter "x" is map_type # map_value:{entries:{key:{string_value:"c"} value:{string_value:"d"}} entries:{key:{string_value:"a"} value:{string_value:"b"}}} Given bindings parameter "x" is MapType({StringType(source='c'): StringType(source='d'), StringType(source='a'): StringType(source='b')}) @@ -1279,7 +1284,8 @@ Scenario: map_ne_right_false Scenario: null_eq_left_true A comparison _==_ against null only binds if the type is determined to be null or we skip the type checking # type:{null:NULL_VALUE} - Given type_env parameter "x" is TypeType(value=None) + # Given type_env parameter "x" is TypeType(value=None) + Given type_env parameter "x" is null_type # null_value:NULL_VALUE Given bindings parameter "x" is None @@ -1292,7 +1298,8 @@ Scenario: null_eq_left_true Scenario: list_eq_right_false # type:{list_type:{elem_type:{primitive:INT64}}} - Given type_env parameter "x" is TypeType(value='list_type') + # Given type_env parameter "x" is TypeType(value='list_type') + Given type_env parameter "x" is list_type # list_value:{values:{int64_value:2} values:{int64_value:1}} Given bindings parameter "x" is [IntType(source=2), IntType(source=1)] @@ -1305,7 +1312,8 @@ Scenario: list_eq_right_false Scenario: string_gte_right_true # type:{primitive:STRING} - Given type_env parameter "x" is TypeType(value='STRING') + # Given type_env parameter "x" is TypeType(value='STRING') + Given type_env parameter "x" is STRING # string_value:"abc" Given bindings parameter "x" is StringType(source='abc') @@ -1318,7 +1326,8 @@ Scenario: string_gte_right_true Scenario: uint_eq_right_false # type:{primitive:UINT64} - Given type_env parameter "x" is TypeType(value='UINT64') + # Given type_env parameter "x" is TypeType(value='UINT64') + Given type_env parameter "x" is UINT64 # uint64_value:1000 Given bindings parameter "x" is UintType(source=1000) @@ -1331,7 +1340,8 @@ Scenario: uint_eq_right_false Scenario: null_lt_right_no_such_overload There is no _<_ operation for null, even if both operands are null # type:{null:NULL_VALUE} - Given type_env parameter "x" is TypeType(value=None) + # Given type_env parameter "x" is TypeType(value=None) + Given type_env parameter "x" is null_type # null_value:NULL_VALUE Given bindings parameter "x" is None diff --git a/features/conversions.feature b/features/conversions.feature index da8b8c7..55b0738 100644 --- a/features/conversions.feature +++ b/features/conversions.feature @@ -161,7 +161,8 @@ Scenario: dyn_heterogeneous_list No need to disable type checking. When CEL expression "type(dyn([1, 'one']))" is evaluated # type_value:"list" - Then value is TypeType(value='list') + # Then value is TypeType(value='list') + Then value is celpy.celtypes.ListType @@ -310,35 +311,39 @@ Scenario: bool When CEL expression "type(true)" is evaluated # type_value:"bool" - Then value is TypeType(value='bool') + # Then value is TypeType(value='bool') + Then value is BoolType Scenario: bool_denotation When CEL expression "bool" is evaluated # type_value:"bool" - Then value is TypeType(value='bool') + # Then value is TypeType(value='bool') + Then value is BoolType Scenario: dyn_no_denotation When CEL expression "dyn" is evaluated # errors:{message:"unknown varaible"} - Then eval_error is 'unknown varaible' + Then eval_error is 'unknown variable' Scenario: int When CEL expression "type(0)" is evaluated # type_value:"int" - Then value is TypeType(value='int') + # Then value is TypeType(value='int') + Then value is IntType Scenario: int_denotation When CEL expression "int" is evaluated # type_value:"int" - Then value is TypeType(value='int') + # Then value is TypeType(value='int') + Then value is IntType Scenario: eq_same @@ -352,84 +357,96 @@ Scenario: uint When CEL expression "type(64u)" is evaluated # type_value:"uint" - Then value is TypeType(value='uint') + # Then value is TypeType(value='uint') + Then value is UintType Scenario: uint_denotation When CEL expression "uint" is evaluated # type_value:"uint" - Then value is TypeType(value='uint') + # Then value is TypeType(value='uint') + Then value is UintType Scenario: double When CEL expression "type(3.14)" is evaluated # type_value:"double" - Then value is TypeType(value='double') + # Then value is TypeType(value='double') + Then value is DoubleType Scenario: double_denotation When CEL expression "double" is evaluated # type_value:"double" - Then value is TypeType(value='double') + # Then value is TypeType(value='double') + Then value is DoubleType Scenario: null_type When CEL expression "type(null)" is evaluated # type_value:"null_type" - Then value is TypeType(value='null_type') + # Then value is TypeType(value='null_type') + Then value is NoneType Scenario: null_type_denotation When CEL expression "null_type" is evaluated # type_value:"null_type" - Then value is TypeType(value='null_type') + # Then value is TypeType(value='null_type') + Then value is NoneType Scenario: string When CEL expression "type('foo')" is evaluated # type_value:"string" - Then value is TypeType(value='string') + # Then value is TypeType(value='string') + Then value is StringType Scenario: string_denotation When CEL expression "string" is evaluated # type_value:"string" - Then value is TypeType(value='string') + # Then value is TypeType(value='string') + Then value is StringType Scenario: bytes When CEL expression "type(b'\xff')" is evaluated # type_value:"bytes" - Then value is TypeType(value='bytes') + # Then value is TypeType(value='bytes') + Then value is BytesType Scenario: bytes_denotation When CEL expression "bytes" is evaluated # type_value:"bytes" - Then value is TypeType(value='bytes') + # Then value is TypeType(value='bytes') + Then value is BytesType Scenario: list When CEL expression "type([1, 2, 3])" is evaluated # type_value:"list" - Then value is TypeType(value='list') + # Then value is TypeType(value='list') + Then value is ListType Scenario: list_denotation When CEL expression "list" is evaluated # type_value:"list" - Then value is TypeType(value='list') + # Then value is TypeType(value='list') + Then value is ListType Scenario: lists_monomorphic @@ -443,14 +460,16 @@ Scenario: map When CEL expression "type({4: 16})" is evaluated # type_value:"map" - Then value is TypeType(value='map') + # Then value is TypeType(value='map') + Then value is MapType Scenario: map_denotation When CEL expression "map" is evaluated # type_value:"map" - Then value is TypeType(value='map') + # Then value is TypeType(value='map') + Then value is MapType Scenario: map_monomorphic @@ -492,21 +511,24 @@ Scenario: type When CEL expression "type(int)" is evaluated # type_value:"type" - Then value is TypeType(value='type') + # Then value is TypeType(value='type') + Then value is TypeType Scenario: type_denotation When CEL expression "type" is evaluated # type_value:"type" - Then value is TypeType(value='type') + # Then value is TypeType(value='type') + Then value is TypeType Scenario: type_type When CEL expression "type(type)" is evaluated # type_value:"type" - Then value is TypeType(value='type') + # Then value is TypeType(value='type') + Then value is TypeType diff --git a/features/environment.py b/features/environment.py index e2988e4..410d889 100644 --- a/features/environment.py +++ b/features/environment.py @@ -3,6 +3,7 @@ """ from functools import partial +import os from types import SimpleNamespace from unittest.mock import Mock, patch @@ -10,15 +11,23 @@ def mock_text_from(context, url): - """Mock for :py:func:`celpy.c7nlib.text_from` that fetches the result from the context""" + """ + Mock for :py:func:`celpy.c7nlib.text_from` that replaces a URL-based request + with a value provided as part of the test context. + """ return context.value_from_data.get(url) def before_scenario(context, scenario): """ - Be sure there's a place to store test scenario data. + Be sure there's a place to store test scenario files. Also. Inject an implementation of the low-level :py:func:`celpy.c7nlib.text_from` function that reads from data provided here. + + Check for command-line or environment option to pick the Runner to be used. + + Use ``-D runner=interpreted`` or ``compiled`` + Or set environment variable ``CEL_RUNNER=interpreted`` or ``compiled`` """ # context.data used by the CEL conformance test suite converted from textproto. context.data = {} @@ -28,6 +37,19 @@ def before_scenario(context, scenario): context.data['container'] = "" # If set, can associate a type binding from local proto files. context.data['json'] = [] + RUNNERS = {"interpreted": celpy.InterpretedRunner, "compiled": celpy.CompiledRunner} + try: + context.data['runner'] = RUNNERS[os.environ.get("CEL_RUNNER", "interpreted")] + except KeyError: + print(f"CEL_RUNNER= must be from {RUNNERS.keys()}") + raise + if "runner" in context.config.userdata: + try: + context.data['runner'] = RUNNERS[context.config.userdata["runner"]] + except KeyError: + print(f"-D runner= must be from {RUNNERS.keys()}") + raise + # context.cel used by the integration test suite. context.cel = {} diff --git a/features/fields.feature b/features/fields.feature index 20615c7..2d47f9b 100644 --- a/features/fields.feature +++ b/features/fields.feature @@ -42,7 +42,8 @@ Scenario: map_key_mix_type Scenario: map_field_access # type:{map_type:{key_type:{primitive:STRING} value_type:{primitive:INT64}}} - Given type_env parameter "x" is TypeType(value='map_type') + # Given type_env parameter "x" is TypeType(value='map_type') + Given type_env parameter "x" is map_type # map_value:{entries:{key:{string_value:"name"} value:{int64_value:1024}}} Given bindings parameter "x" is MapType({StringType(source='name'): IntType(source=1024)}) @@ -62,7 +63,8 @@ Scenario: map_no_such_key Scenario: map_field_select_no_such_key # type:{map_type:{key_type:{primitive:STRING} value_type:{primitive:STRING}}} - Given type_env parameter "x" is TypeType(value='map_type') + # Given type_env parameter "x" is TypeType(value='map_type') + Given type_env parameter "x" is map_type # map_value:{entries:{key:{string_value:"holiday"} value:{string_value:"field"}}} Given bindings parameter "x" is MapType({StringType(source='holiday'): StringType(source='field')}) @@ -172,7 +174,8 @@ Scenario: has_empty Scenario: qualified_ident # type:{primitive:STRING} - Given type_env parameter "a.b.c" is TypeType(value='STRING') + # Given type_env parameter "a.b.c" is TypeType(value='STRING') + Given type_env parameter "a.b.c" is STRING # string_value:"yeah" Given bindings parameter "a.b.c" is StringType(source='yeah') @@ -185,7 +188,8 @@ Scenario: qualified_ident Scenario: map_field_select # type:{map_type:{key_type:{primitive:STRING} value_type:{primitive:STRING}}} - Given type_env parameter "a.b" is TypeType(value='map_type') + # Given type_env parameter "a.b" is TypeType(value='map_type') + Given type_env parameter "a.b" is map_type # map_value:{entries:{key:{string_value:"c"} value:{string_value:"yeah"}}} Given bindings parameter "a.b" is MapType({StringType(source='c'): StringType(source='yeah')}) @@ -198,10 +202,12 @@ Scenario: map_field_select Scenario: qualified_identifier_resolution_unchecked namespace resolution should try to find the longest prefix for the evaluator. # type:{primitive:STRING} - Given type_env parameter "a.b.c" is TypeType(value='STRING') + # Given type_env parameter "a.b.c" is TypeType(value='STRING') + Given type_env parameter "a.b.c" is STRING # type:{map_type:{key_type:{primitive:STRING} value_type:{primitive:STRING}}} - Given type_env parameter "a.b" is TypeType(value='map_type') + # Given type_env parameter "a.b" is TypeType(value='map_type') + Given type_env parameter "a.b" is map_type # map_value:{entries:{key:{string_value:"c"} value:{string_value:"oops"}}} Given bindings parameter "a.b" is MapType({StringType(source='c'): StringType(source='oops')}) @@ -217,7 +223,8 @@ Scenario: qualified_identifier_resolution_unchecked Scenario: list_field_select_unsupported # type:{list_type:{elem_type:{primitive:STRING}}} - Given type_env parameter "a.b" is TypeType(value='list_type') + # Given type_env parameter "a.b" is TypeType(value='list_type') + Given type_env parameter "a.b" is list_type # list_value:{values:{string_value:"pancakes"}} Given bindings parameter "a.b" is [StringType(source='pancakes')] @@ -230,7 +237,8 @@ Scenario: list_field_select_unsupported Scenario: int64_field_select_unsupported # type:{primitive:INT64} - Given type_env parameter "a" is TypeType(value='INT64') + # Given type_env parameter "a" is TypeType(value='INT64') + Given type_env parameter "a" is INT64 # int64_value:15 Given bindings parameter "a" is IntType(source=15) @@ -243,10 +251,12 @@ Scenario: int64_field_select_unsupported Scenario: ident_with_longest_prefix_check namespace resolution should try to find the longest prefix for the checker. # type:{primitive:STRING} - Given type_env parameter "a.b.c" is TypeType(value='STRING') + # Given type_env parameter "a.b.c" is TypeType(value='STRING') + Given type_env parameter "a.b.c" is STRING # type:{map_type:{key_type:{primitive:STRING} value_type:{primitive:STRING}}} - Given type_env parameter "a.b" is TypeType(value='map_type') + # Given type_env parameter "a.b" is TypeType(value='map_type') + Given type_env parameter "a.b" is map_type # map_value:{entries:{key:{string_value:"c"} value:{string_value:"oops"}}} Given bindings parameter "a.b" is MapType({StringType(source='c'): StringType(source='oops')}) diff --git a/features/namespace.feature b/features/namespace.feature index 1a10de8..83bd11d 100644 --- a/features/namespace.feature +++ b/features/namespace.feature @@ -7,7 +7,8 @@ Feature: namespace Scenario: self_eval_qualified_lookup # type:{primitive:BOOL} - Given type_env parameter "x.y" is TypeType(value='BOOL') + # Given type_env parameter "x.y" is TypeType(value='BOOL') + Given type_env parameter "x.y" is BOOL # bool_value:true Given bindings parameter "x.y" is BoolType(source=True) @@ -23,10 +24,12 @@ Scenario: self_eval_qualified_lookup Scenario: self_eval_container_lookup # type:{primitive:BOOL} - Given type_env parameter "x.y" is TypeType(value='BOOL') + # Given type_env parameter "x.y" is TypeType(value='BOOL') + Given type_env parameter "x.y" is BOOL # type:{primitive:STRING} - Given type_env parameter "y" is TypeType(value='STRING') + # Given type_env parameter "y" is TypeType(value='STRING') + Given type_env parameter "y" is STRING # bool_value:true Given bindings parameter "x.y" is BoolType(source=True) @@ -44,10 +47,12 @@ Scenario: self_eval_container_lookup Scenario: self_eval_container_lookup_unchecked # type:{primitive:BOOL} - Given type_env parameter "x.y" is TypeType(value='BOOL') + # Given type_env parameter "x.y" is TypeType(value='BOOL') + Given type_env parameter "x.y" is BOOL # type:{primitive:BOOL} - Given type_env parameter "y" is TypeType(value='BOOL') + # Given type_env parameter "y" is TypeType(value='BOOL') + Given type_env parameter "y" is BOOL # bool_value:true Given bindings parameter "x.y" is BoolType(source=True) diff --git a/features/parse.feature b/features/parse.feature index b4159a2..d7694d2 100644 --- a/features/parse.feature +++ b/features/parse.feature @@ -7,7 +7,8 @@ Feature: parse Scenario: list_index Member = Member '[' Expr ']' # type:{list_type:{elem_type:{primitive:INT64}}} - Given type_env parameter "a" is TypeType(value='list_type') + # Given type_env parameter "a" is TypeType(value='list_type') + Given type_env parameter "a" is list_type # list_value:{values:{int64_value:0}} Given bindings parameter "a" is [IntType(source=0)] diff --git a/features/proposed.feature b/features/proposed.feature index 5822058..dd9fd90 100644 --- a/features/proposed.feature +++ b/features/proposed.feature @@ -1,3 +1,4 @@ +@future Feature: proposed features, not yet part of CEL See https://github.com/google/cel-spec/issues/143 for the new macros - .reduce(, , , ) diff --git a/features/steps/cli_binding.py b/features/steps/cli_binding.py index b57a83a..6b7fbe3 100644 --- a/features/steps/cli_binding.py +++ b/features/steps/cli_binding.py @@ -51,9 +51,6 @@ def step_impl(context, arguments): else: environment = {} environment.update(context.data['bindings']) - # if sys.version_info.minor <= 6: - # extra = {} - # else: extra = {'text': True} context.data['arguments'] = shlex.split(arguments) @@ -77,10 +74,6 @@ def step_impl(context, arguments): test_dir.rmdir() context.data['status'] = result.returncode - # if sys.version_info.minor <= 6: - # context.data['stdout'] = result.stdout.decode('utf-8') - # context.data['stderr'] = result.stderr.decode('utf-8') - # else: context.data['stdout'] = result.stdout context.data['stderr'] = result.stderr diff --git a/features/steps/integration_binding.py b/features/steps/integration_binding.py index a7ee442..6990170 100644 --- a/features/steps/integration_binding.py +++ b/features/steps/integration_binding.py @@ -56,14 +56,16 @@ Use ``-D match=exact`` to do exact error matching. The default is "any error will do." """ import logging -import re -import subprocess import sys from enum import Enum, auto from pathlib import Path +try: + from types import NoneType +except ImportError: + # Python 3.9 hack + NoneType = type(None) from typing import (Any, Callable, Dict, List, NamedTuple, Optional, Tuple, Type, Union, cast) -from unittest.mock import MagicMock, Mock from behave import * @@ -85,20 +87,6 @@ class TypeKind(str, Enum): TYPE_SPEC = "type_spec" -# class TypeEnv(NamedTuple): -# name: str # The variable for which we're providing a type -# kind: TypeKind # Not too useful, except for MAP_TYPE -# type_ident: Union[str, List[str]] # The type(s) to define -# -# @property -# def annotation(self) -> Tuple[str, Callable]: -# """Translate Protobuf/Gherkin TypeEnv to a CEL type declaration""" -# if self.kind == TypeKind.MAP_TYPE: -# # A mapping, constrained by the self.type_ident array. -# return (self.name, f"Map[{', '.join(self.type_ident)}]") -# return (self.name, self.type_ident) - - class Bindings(NamedTuple): bindings: List[Dict[str, Any]] @@ -320,14 +308,18 @@ def get(self, field: Any, default: Optional[Value] = None) -> Value: Provides default values for the defined fields. """ logger.info(f"NestedTestAllTypes.get({field!r}, {default!r})") - if field == "child": - default_class = NestedTestAllTypes + if field in self: + return self[field] + elif field == "child": + return NestedTestAllTypes() elif field == "payload": - default_class = TestAllTypes + return TestAllTypes() + elif default is not None: + return default else: err = f"no such member in mapping: {field!r}" raise KeyError(err) - return super().get(field, default if default is not None else default_class()) + # return super().get(field, default if default is not None else default_class()) class NestedMessage(celpy.celtypes.MessageType): """ @@ -349,6 +341,45 @@ class NestedMessage(celpy.celtypes.MessageType): """ pass +# From Protobuf definitions, these are the CEL types implement them. +TYPE_NAMES = { + "google.protobuf.Any": MessageType, + "google.protubuf.Any": MessageType, # Note spelling anomaly. + "google.protobuf.BoolValue": BoolType, + "google.protobuf.BytesValue": BytesType, + "google.protobuf.DoubleValue": DoubleType, + "google.protobuf.Duration": DurationType, + "google.protobuf.FloatValue": DoubleType, + "google.protobuf.Int32Value": IntType, + "google.protobuf.Int64Value": IntType, + "google.protobuf.ListValue": ListType, + "google.protobuf.StringValue": StringType, + "google.protobuf.Struct": MessageType, + "google.protobuf.Timestamp": TimestampType, + "google.protobuf.UInt32Value": UintType, + "google.protobuf.UInt64Value": UintType, + "google.protobuf.Value": MessageType, + "type": TypeType, + "list_type": ListType, + "map_type": MapType, + "map": MapType, + "list": ListType, + "string": StringType, + "bytes": BytesType, + "bool": BoolType, + "int": IntType, + "uint": UintType, + "double": DoubleType, + "null_type": NoneType, + "STRING": StringType, + "BOOL": BoolType, + "INT64": IntType, + "UINT64": UintType, + "INT32": IntType, + "UINT32": UintType, + "BYTES": BytesType, + "DOUBLE": DoubleType, +} @given(u'disable_check parameter is {disable_check}') def step_impl(context, disable_check): @@ -359,9 +390,17 @@ def step_impl(context, disable_check): def step_impl(context, name, type_env): """ type_env has name and type information used to create the environment. + Generally, it should be one of the type names, e.g. ``INT64``. + These need to be mapped to celpy.celtypes types. + + Sometimes it already is a ``celpy.celtypes`` name. """ - type_value = eval(type_env) - context.data['type_env'][name] = type_value + if type_env.startswith("celpy"): + context.data['type_env'][name] = eval(type_env) + if type_env.startswith('"'): + context.data['type_env'][name] = TYPE_NAMES[type_env[1:-1]] + else: + context.data['type_env'][name] = TYPE_NAMES[type_env] @given(u'bindings parameter "{name}" is {binding}') @@ -395,7 +434,10 @@ def cel(context): context.data['test_all_types'] = TestAllTypes context.data['nested_test_all_types'] = NestedTestAllTypes - env = Environment(package=context.data['container'], annotations=context.data['type_env']) + env = Environment( + package=context.data['container'], + annotations=context.data['type_env'], + runner_class=context.data['runner']) ast = env.compile(context.data['expr']) prgm = env.program(ast) @@ -404,9 +446,11 @@ def cel(context): try: result = prgm.evaluate(activation) context.data['result'] = result + context.data['exc_info'] = None context.data['error'] = None except CELEvalError as ex: # No 'result' to distinguish from an expected None value. + context.data['exc_info'] = sys.exc_info() context.data['error'] = ex.args[0] @@ -425,21 +469,19 @@ def step_impl(context, expr): @then(u'value is {value}') def step_impl(context, value): """ - value should be the repr() string for a CEL object. - - We make a special case around textproto ``{ type_value: "google.protobuf.Duration" }``. - These come to us as a ``TypeType(value='google.protobuf.Duration')`` as an expected result. + The ``value`` **must** be the ``repr()`` string for a CEL object. - We don't actually create a protobuf objects, but we've defined TypeType to match - CELType classes against protobuf type names. + This includes types and protobuf messages. """ try: expected = eval(value) except TypeError as ex: - print(f"Could not eval({value!r})") + print(f"Could not eval({value!r}) in {context.scenario}") raise context.data['expected'] = expected - assert 'result' in context.data, f"Error {context.data['error']!r}; no result in {context.data!r}" + if 'result' not in context.data: + print("Unexpected exception:", context.data['exc_info']) + raise AssertionError(f"Error {context.data['error']!r} unexpected") result = context.data['result'] if expected is not None: assert result == expected, \ @@ -450,33 +492,65 @@ def step_impl(context, value): class ErrorCategory(Enum): divide_by_zero = auto() + does_not_support = auto() + integer_overflow = auto() + invalid = auto() + invalid_argument = auto() modulus_by_zero = auto() + no_such_key = auto() + no_such_member = auto() no_such_overload = auto() - integer_overflow = auto() + range_error = auto() + repeated_key = auto() + unbound_function = auto() undeclared_reference = auto() unknown_variable = auto() + other = auto() ERROR_ALIASES = { "division by zero": ErrorCategory.divide_by_zero, "divide by zero": ErrorCategory.divide_by_zero, + "invalid UTF-8": ErrorCategory.invalid, "modulus by zero": ErrorCategory.modulus_by_zero, + "modulus or divide by zero": ErrorCategory.modulus_by_zero, + "no such key": ErrorCategory.no_such_key, + "no such member": ErrorCategory.no_such_member, "no such overload": ErrorCategory.no_such_overload, "no matching overload": ErrorCategory.no_such_overload, + "range": ErrorCategory.range_error, + "range error": ErrorCategory.range_error, + "repeated key": ErrorCategory.repeated_key, + "Failed with repeated key": ErrorCategory.repeated_key, "return error for overflow": ErrorCategory.integer_overflow, - "unknown varaible": ErrorCategory.unknown_variable, + "unknown variable": ErrorCategory.unknown_variable, + "unknown varaible": ErrorCategory.unknown_variable, # spelling error in TextProto + "unbound function": ErrorCategory.unbound_function, + "unsupported key type": ErrorCategory.does_not_support, } def error_category(text: str) -> ErrorCategory: + """Summarize errors into broad ErrorCategory groupings.""" if text in ErrorCategory.__members__: return ErrorCategory[text] if text in ERROR_ALIASES: return ERROR_ALIASES[text] - # The hard problem: "undeclared reference to 'x' (in container '')" - if text.startswith("undeclared reference"): - return ErrorCategory.undeclared_reference + # Some harder problems: + if text.startswith("undeclared reference to"): + return ErrorCategory.undeclared_reference + elif text.startswith("found no matching overload for"): + return ErrorCategory.no_such_overload + elif text.startswith("no such key"): + return ErrorCategory.no_such_key + elif text.startswith("no such member"): + return ErrorCategory.no_such_member + elif "does not support" in text: + return ErrorCategory.does_not_support + else: + print(f"***No error category for {text!r}***") + return ErrorCategory.other @then(u"eval_error is {quoted_text}") def step_impl(context, quoted_text): @@ -491,9 +565,10 @@ def step_impl(context, quoted_text): if quoted_text == "None": assert context.data['error'] is None, f"error not None in {context.data}" else: + print(f"*** Analyzing context.data = {context.data!r}***") text = quoted_text[1:-1] if quoted_text[0] in ["'", '"'] else quoted_text - actual_ec = error_category(context.data['error'] or "") expected_ec = error_category(text) + actual_ec = error_category(context.data['error'] or "") if context.config.userdata.get("match", "any") == "exact": assert expected_ec == actual_ec, f"{expected_ec} != {actual_ec} in {context.data}" else: diff --git a/features/timestamps.feature b/features/timestamps.feature index ab51cc5..20ab516 100644 --- a/features/timestamps.feature +++ b/features/timestamps.feature @@ -22,7 +22,8 @@ Scenario: toType_timestamp When CEL expression "type(timestamp('2009-02-13T23:31:30Z'))" is evaluated # type_value:"google.protobuf.Timestamp" - Then value is TypeType(value='google.protobuf.Timestamp') + # Then value is TypeType(value='google.protobuf.Timestamp') + Then value is TimestampType @@ -39,7 +40,8 @@ Scenario: toType_duration When CEL expression "type(duration('1000000s'))" is evaluated # type_value:"google.protobuf.Duration" - Then value is TypeType(value='google.protobuf.Duration') + # Then value is TypeType(value='google.protobuf.Duration') + Then value is DurationType @@ -409,7 +411,8 @@ Scenario: get_hours Scenario: get_milliseconds Need to import a variable to get milliseconds. # type:{message_type:"google.protobuf.Duration"} - Given type_env parameter "x" is TypeType(value='google.protobuf.Duration') + # Given type_env parameter "x" is TypeType(value='google.protobuf.Duration') + Given type_env parameter "x" is google.protobuf.Duration # object_value:{[type.googleapis.com/google.protobuf.Duration]:{seconds:123 nanos:123456789}} Given bindings parameter "x" is DurationType(seconds=123, nanos=123456789) diff --git a/pyproject.toml b/pyproject.toml index 776577b..fdeaf7d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -34,7 +34,6 @@ dependencies = [ "pyyaml>=6.0.2", "jmespath>=1.0.1", "tomli >= 1.1.0 ; python_version < '3.11'", - "sphinx>=7.4.7", ] [project.scripts] @@ -59,8 +58,10 @@ dev = [ "mypy>=1.15.0", "pytest>=8.3.5", "ruff>=0.11.10", + "sphinx>=7.4.7", "tox>=4.24", "tox-uv>=1.25.0", "types-pyyaml>=6.0.12.20250516", "google-re2-stubs", + "sphinxcontrib-plantuml>=0.30", ] diff --git a/src/celpy/__init__.py b/src/celpy/__init__.py index 031f6d2..5b8e592 100644 --- a/src/celpy/__init__.py +++ b/src/celpy/__init__.py @@ -14,91 +14,40 @@ # See the License for the specific language governing permissions and limitations under the License. """ -Pure Python implementation of CEL. - -.. todo:: Consolidate __init__ and parser into one module? - -Visible interface to CEL. This exposes the :py:class:`Environment`, -the :py:class:`Evaluator` run-time, and the :py:mod:`celtypes` module -with Python types wrapped to be CEL compatible. - -Example -======= - -Here's an example with some details:: - - >>> import celpy - - # A list of type names and class bindings used to create an environment. - >>> types = [] - >>> env = celpy.Environment(types) - - # Parse the code to create the CEL AST. - >>> ast = env.compile("355. / 113.") - - # Use the AST and any overriding functions to create an executable program. - >>> functions = {} - >>> prgm = env.program(ast, functions) - - # Variable bindings. - >>> activation = {} - - # Final evaluation. - >>> try: - ... result = prgm.evaluate(activation) - ... error = None - ... except CELEvalError as ex: - ... result = None - ... error = ex.args[0] - - >>> result # doctest: +ELLIPSIS - DoubleType(3.14159...) - -Another Example -=============== - -See https://github.com/google/cel-go/blob/master/examples/simple_test.go - -The model Go we're sticking close to:: - - d := cel.Declarations(decls.NewVar("name", decls.String)) - env, err := cel.NewEnv(d) - if err != nil { - log.Fatalf("environment creation error: %v\\n", err) - } - ast, iss := env.Compile(`"Hello world! I'm " + name + "."`) - // Check iss for compilation errors. - if iss.Err() != nil { - log.Fatalln(iss.Err()) - } - prg, err := env.Program(ast) - if err != nil { - log.Fatalln(err) - } - out, _, err := prg.Eval(map[string]interface{}{ - "name": "CEL", - }) - if err != nil { - log.Fatalln(err) - } - fmt.Println(out) - // Output:Hello world! I'm CEL. - -Here's the Pythonic approach, using concept patterned after the Go implementation:: - - >>> from celpy import * - >>> decls = {"name": celtypes.StringType} - >>> env = Environment(annotations=decls) - >>> ast = env.compile('"Hello world! I\\'m " + name + "."') - >>> out = env.program(ast).evaluate({"name": "CEL"}) - >>> print(out) - Hello world! I'm CEL. +The pure Python implementation of the Common Expression Language, CEL. +This module defines an interface to CEL for integration into other Python applications. +This exposes the :py:class:`Environment` used to compile the source module, +the :py:class:`Runner` used to evaluate the compiled code, +and the :py:mod:`celpy.celtypes` module with Python types wrapped to be CEL compatible. + +The way these classes are used is as follows: + +.. uml:: + + @startuml + start + :Gather (or define) annotations; + :Create ""Environment""; + :Compile CEL; + :Create ""Runner"" with any extension functions; + :Evaluate the ""Runner"" with a ""Context""; + stop + @enduml + +The explicit decomposition into steps permits +two extensions: + +1. Transforming the AST to introduce any optimizations. + +2. Saving the :py:class:`Runner` instance to reuse an expression with new inputs. """ +import abc import json # noqa: F401 import logging import sys +from textwrap import indent from typing import Any, Dict, Optional, Type, cast import lark @@ -118,6 +67,8 @@ Context, Evaluator, Result, + Transpiler, + TranspilerTree, base_functions, ) @@ -125,89 +76,129 @@ Expression = lark.Tree -class Runner: - """Abstract runner. +class Runner(abc.ABC): + """Abstract runner for a compiled CEL program. + + The :py:class:`Environment` creates a :py:class:`Runner` to permit + saving a ready-tp-evaluate, compiled CEL expression. + A :py:class:`Runner` will evaluate the AST in the context of a specific activation + with the provided variable values. + + A :py:class:`Runner` class provides + the ``tree_node_class`` attribute to define the type for tree nodes. + This class information is used by the :py:class:`Environment` to tailor the ``Lark`` instance created. + The class named by the ``tree_node_class`` can include specialized AST features + needed by a :py:class:`Runner` instance. - Given an AST, this can evaluate the AST in the context of a specific activation - with any override function definitions. + .. todo:: For a better fit with Go language expectations - .. todo:: add type adapter and type provider registries. + Consider addoing type adapter and type provider registries. + This would permit distinct sources of protobuf message types. """ + tree_node_class: type = lark.Tree + def __init__( self, environment: "Environment", ast: lark.Tree, functions: Optional[Dict[str, CELFunction]] = None, ) -> None: + """ + Initialize this ``Runner`` with a given AST. + The Runner will have annotations take from the :py:class:`Environment`, + plus any unique functions defined here. + """ self.logger = logging.getLogger(f"celpy.{self.__class__.__name__}") self.environment = environment self.ast = ast self.functions = functions - def new_activation(self, context: Context) -> Activation: + def __repr__(self) -> str: + return f"{self.__class__.__name__}({self.environment}, {self.ast}, {self.functions})" + + def new_activation(self) -> Activation: """ - Builds the working activation from the environmental defaults. + Builds a new, working :py:class:`Activation` using the :py:class:`Environment` as defaults. + A Context will later be layered onto this for evaluation. """ - return self.environment.activation().nested_activation(vars=context) + base_activation = Activation( + package=self.environment.package, + annotations=self.environment.annotations, + functions=self.functions, + ) + return base_activation + @abc.abstractmethod def evaluate(self, activation: Context) -> celpy.celtypes.Value: # pragma: no cover - raise NotImplementedError + """ + Given variable definitions in the :py:class:`celpy.evaluation.Context`, evaluate the given AST and return the resulting value. + + Generally, this should raise an :exc:`CELEvalError` for most kinds of ordinary problems. + It may raise an :exc:`CELUnsupportedError` for future features that aren't fully implemented. + Any Python exception reflects a serious problem. + """ + ... class InterpretedRunner(Runner): """ - Pure AST expression evaluator. Uses :py:class:`evaluation.Evaluator` class. - - Given an AST, this evauates the AST in the context of a specific activation. - - The returned value will be a celtypes type. - - Generally, this should raise an :exc:`CELEvalError` for most kinds of ordinary problems. - It may raise an :exc:`CELUnsupportedError` for future features. - - .. todo:: Refractor the Evaluator constructor from evaluation. + An **Adapter** for the :py:class:`celpy.evaluation.Evaluator` class. """ def evaluate(self, context: Context) -> celpy.celtypes.Value: e = Evaluator( ast=self.ast, - activation=self.new_activation(context), - functions=self.functions, + activation=self.new_activation(), ) - value = e.evaluate() + value = e.evaluate(context) return value class CompiledRunner(Runner): """ - Python compiled expression evaluator. Uses Python byte code and :py:func:`eval`. + An **Adapter** for the :py:class:`celpy.evaluation.Transpiler` class. - Given an AST, this evaluates the AST in the context of a specific activation. + A :py:class:`celpy.evaluation.Transpiler` instance transforms the AST into Python. + It uses :py:func:`compile` to create a code object. + The final :py:meth:`evaluate` method uses :py:func:`exec` to evaluate the code object. - Transform the AST into Python, uses :py:func:`compile` to create a code object. - Uses :py:func:`eval` to evaluate. + Note, this requires the ``celpy.evaluation.TranspilerTree`` classes + instead of the default ``lark.Tree`` class. """ + tree_node_class: type = TranspilerTree + def __init__( self, environment: "Environment", - ast: lark.Tree, + ast: TranspilerTree, functions: Optional[Dict[str, CELFunction]] = None, ) -> None: + """ + Transpile to Python, and use :py:func:`compile` to create a code object. + """ super().__init__(environment, ast, functions) - # Transform AST to Python. - # compile() - # cache executable code object. + self.tp = Transpiler( + ast=cast(TranspilerTree, self.ast), + activation=self.new_activation(), + ) + self.tp.transpile() + self.logger.info("Transpiled:\n%s", indent(self.tp.source_text, " ")) - def evaluate(self, activation: Context) -> celpy.celtypes.Value: - # eval() code object with activation as locals, and built-ins as gobals. - return super().evaluate(activation) + def evaluate(self, context: Context) -> celpy.celtypes.Value: + """ + Use :py:func:`exec` to execute the code object. + """ + value = self.tp.evaluate(context) + return value -# TODO: Refactor classes into a separate "cel_protobuf" module. -# TODO: Becomes cel_protobuf.Int32Value +# TODO: Refactor this class into a separate "cel_protobuf" module. +# TODO: Rename this type to ``cel_protobuf.Int32Value`` class Int32Value(celpy.celtypes.IntType): + """A wrapper for int32 values.""" + def __new__( cls: Type["Int32Value"], value: Any = 0, @@ -221,8 +212,8 @@ def __new__( return cast(Int32Value, super().__new__(cls, convert(value))) -# The "well-known" types in a google.protobuf package. -# We map these to CEl types instead of defining additional Protobuf Types. +# The "well-known" types in a ``google.protobuf`` package. +# We map these to CEL types instead of defining additional Protobuf Types. # This approach bypasses some of the range constraints that are part of these types. # It may also cause values to compare as equal when they were originally distinct types. googleapis = { @@ -241,15 +232,19 @@ def __new__( class Environment: - """Compiles CEL text to create an Expression object. + """ + Contains the current evaluation context. + The :py:meth:`Environment.compile` method + compiles CEL text to create an AST. - From the Go implementation, there are things to work with the type annotations: + The :py:meth:`Environment.program` method + packages the AST into a program ready for evaluation. - - type adapters registry make other native types available for CEL. + .. todo:: For a better fit with Go language expectations - - type providers registry make ProtoBuf types available for CEL. + - A type adapters registry makes other native types available for CEL. - .. todo:: Add adapter and provider registries to the Environment. + - A type providers registry make ProtoBuf types available for CEL. """ def __init__( @@ -278,7 +273,7 @@ def __init__( self.annotations: Dict[str, Annotation] = annotations or {} self.logger.debug("Type Annotations %r", self.annotations) self.runner_class: Type[Runner] = runner_class or InterpretedRunner - self.cel_parser = CELParser() + self.cel_parser = CELParser(tree_class=self.runner_class.tree_node_class) self.runnable: Runner # Fold in standard annotations. These (generally) define well-known protobuf types. @@ -286,6 +281,9 @@ def __init__( # We'd like to add 'type.googleapis.com/google' directly, but it seems to be an alias # for 'google', the path after the '/' in the uri. + def __repr__(self) -> str: + return f"{self.__class__.__name__}({self.package}, {self.annotations}, {self.runner_class})" + def compile(self, text: str) -> Expression: """Compile the CEL source. This can raise syntax error exceptions.""" ast = self.cel_parser.parse(text) @@ -294,13 +292,15 @@ def compile(self, text: str) -> Expression: def program( self, expr: lark.Tree, functions: Optional[Dict[str, CELFunction]] = None ) -> Runner: - """Transforms the AST into an executable runner.""" + """ + Transforms the AST into an executable :py:class:`Runner` object. + + :param expr: The parse tree from :py:meth:`compile`. + :param functions: Any additional functions to be used by this CEL expression. + :returns: A :py:class:`Runner` instance that can be evaluated with a ``Context`` that provides values. + """ self.logger.debug("Package %r", self.package) runner_class = self.runner_class self.runnable = runner_class(self, expr, functions) + self.logger.debug("Runnable %r", self.runnable) return self.runnable - - def activation(self) -> Activation: - """Returns a base activation""" - activation = Activation(package=self.package, annotations=self.annotations) - return activation diff --git a/src/celpy/__main__.py b/src/celpy/__main__.py index 30eb45c..0b89173 100644 --- a/src/celpy/__main__.py +++ b/src/celpy/__main__.py @@ -14,85 +14,25 @@ # See the License for the specific language governing permissions and limitations under the License. """ -Pure Python implementation of CEL. - -This provides a few jq-like, bc-like, and shell expr-like features. - -- ``jq`` uses ``.`` to refer the current document. By setting a package - name of ``"jq"`` and placing the JSON object in the package, we achieve - similar syntax. - -- ``bc`` offers complex function definitions and other programming support. - CEL can only evaluate a few bc-like expressions. - -- This does everything ``expr`` does, but the syntax is slightly different. - The output of comparisons -- by default -- is boolean, where ``expr`` is an integer 1 or 0. - Use ``-f 'd'`` to see decimal output instead of Boolean text values. - -- This does some of what ``test`` does, without a lot of the sophisticated - file system data gathering. - Use ``-b`` to set the exit status code from a Boolean result. - -TODO: This can also have a REPL, as well as process CSV files. - -SYNOPSIS -======== - -:: - - python -m celpy [--arg name:type=value ...] [--null-input] expr - -Options: - -:--arg: - Provides argument names, types and optional values. - If the value is not provided, the name is expected to be an environment - variable, and the value of the environment variable is converted and used. - -:--null-input: - Normally, JSON documents are read from stdin in ndjson format. If no JSON documents are - provided, the ``--null-input`` option skips trying to read from stdin. - -:expr: - A CEL expression to evaluate. - -JSON documents are read from stdin in NDJSON format (http://jsonlines.org/, http://ndjson.org/). -For each JSON document, the expression is evaluated with the document in a default -package. This allows `.name` to pick items from the document. - -By default, the output is JSON serialized. This means strings will be JSON-ified and have quotes. - -If a ``--format`` option is provided, this is applied to the resulting object; this can be -used to strip quotes, or limit precision on double objects, or convert numbers to hexadecimal. - -Arguments, Types, and Namespaces -================================ - -CEL objects rely on the celtypes definitions. - -Because of the close association between CEL and protobuf, some well-known protobuf types -are also supported. - -.. todo:: CLI type environment - - Permit name.name:type=value to create namespace bindings. - -Further, type providers can be bound to CEL. This means an extended CEL -may have additional types beyond those defined by the :py:class:`Activation` class. +The CLI interface to ``celpy``. +This parses the command-line options. +It also offers an interactive REPL. """ import argparse import ast import cmd +import datetime import json import logging import logging.config import os from pathlib import Path import re +import stat as os_stat import sys -from typing import Any, Callable, Dict, List, Optional, Tuple, cast +from typing import Any, Callable, Dict, List, Optional, Tuple, Union, cast try: import tomllib @@ -188,7 +128,7 @@ def arg_type_value(text: str) -> Tuple[str, Annotation, celtypes.Value]: type_definition = CLI_ARG_TYPES[type_name] value = cast( celtypes.Value, - type_definition(value_text), # type: ignore[arg-type, call-arg] + type_definition(value_text), # type: ignore [call-arg] ) except KeyError: raise argparse.ArgumentTypeError( @@ -295,6 +235,73 @@ def get_options(argv: Optional[List[str]] = None) -> argparse.Namespace: return options +def stat(path: Union[Path, str]) -> Optional[celtypes.MapType]: + """This function is added to the CLI to permit file-system interrogation.""" + try: + status = Path(path).stat() + data = { + "st_atime": celtypes.TimestampType( + datetime.datetime.fromtimestamp(status.st_atime) + ), + "st_ctime": celtypes.TimestampType( + datetime.datetime.fromtimestamp(status.st_ctime) + ), + "st_mtime": celtypes.TimestampType( + datetime.datetime.fromtimestamp(status.st_mtime) + ), + "st_dev": celtypes.IntType(status.st_dev), + "st_ino": celtypes.IntType(status.st_ino), + "st_nlink": celtypes.IntType(status.st_nlink), + "st_size": celtypes.IntType(status.st_size), + "group_access": celtypes.BoolType(status.st_gid == os.getegid()), + "user_access": celtypes.BoolType(status.st_uid == os.geteuid()), + } + + # From mode File type: + # - block, character, directory, regular, symbolic link, named pipe, socket + # One predicate should be True; we want the code for that key. + data["kind"] = celtypes.StringType( + { + predicate(status.st_mode): code + for code, predicate in [ + ("b", os_stat.S_ISBLK), + ("c", os_stat.S_ISCHR), + ("d", os_stat.S_ISDIR), + ("f", os_stat.S_ISREG), + ("p", os_stat.S_ISFIFO), + ("l", os_stat.S_ISLNK), + ("s", os_stat.S_ISSOCK), + ] + }.get(True, "?") + ) + + # Special bits: uid, gid, sticky + data["setuid"] = celtypes.BoolType((os_stat.S_ISUID & status.st_mode) != 0) + data["setgid"] = celtypes.BoolType((os_stat.S_ISGID & status.st_mode) != 0) + data["sticky"] = celtypes.BoolType((os_stat.S_ISVTX & status.st_mode) != 0) + + # permissions, limited to user-level RWX, nothing more. + data["r"] = celtypes.BoolType(os.access(path, os.R_OK)) + data["w"] = celtypes.BoolType(os.access(path, os.W_OK)) + data["x"] = celtypes.BoolType(os.access(path, os.X_OK)) + try: + extra = { + "st_birthtime": celtypes.TimestampType( + datetime.datetime.fromtimestamp(status.st_birthtime) + ), + "st_blksize": celtypes.IntType(status.st_blksize), + "st_blocks": celtypes.IntType(status.st_blocks), + "st_flags": celtypes.IntType(status.st_flags), + "st_rdev": celtypes.IntType(status.st_rdev), + "st_gen": celtypes.IntType(status.st_gen), + } + except AttributeError: # pragma: no cover + extra = {} + return celtypes.MapType(data | extra) + except FileNotFoundError: + return None + + class CEL_REPL(cmd.Cmd): prompt = "CEL> " intro = "Enter an expression to have it evaluated." @@ -341,6 +348,7 @@ def do_quit(self, args: str) -> bool: do_exit = do_quit do_bye = do_quit + do_EOF = do_quit def default(self, args: str) -> None: """Evaluate an expression.""" @@ -369,10 +377,10 @@ def process_json_doc( """ try: activation[variable] = json.loads(document, cls=CELJSONDecoder) - result = prgm.evaluate(activation) - display(result) - if boolean_to_status and isinstance(result, (celtypes.BoolType, bool)): - return 0 if result else 1 + result_value = prgm.evaluate(activation) + display(result_value) + if boolean_to_status and isinstance(result_value, (celtypes.BoolType, bool)): + return 0 if result_value else 1 return 0 except CELEvalError as ex: # ``jq`` KeyError problems result in ``None``. @@ -416,12 +424,12 @@ def main(argv: Optional[List[str]] = None) -> int: if options.format: - def output_display(result: Result) -> None: - print("{0:{format}}".format(result, format=options.format)) + def output_display(result_value: Result) -> None: + print("{0:{format}}".format(result_value, format=options.format)) else: - def output_display(result: Result) -> None: - print(json.dumps(result, cls=CELJSONEncoder)) + def output_display(result_value: Result) -> None: + print(json.dumps(result_value, cls=CELJSONEncoder)) logger.info("Expr: %r", options.expr) @@ -432,17 +440,18 @@ def output_display(result: Result) -> None: if options.arg: annotations = {name: type for name, type, value in options.arg} else: - annotations = None + annotations = {} + annotations["stat"] = celtypes.FunctionType # If we're creating a named JSON document, we don't provide a default package. - # If we're usinga JSON document to populate a package, we provide the given name. + # If we're using a JSON document to populate a package, we provide the given name. env = Environment( package=None if options.null_input else options.package, annotations=annotations, ) try: expr = env.compile(options.expr) - prgm = env.program(expr) + prgm = env.program(expr, functions={"stat": stat}) except CELParseError as ex: print( env.cel_parser.error_text(ex.args[0], ex.line, ex.column), file=sys.stderr @@ -457,17 +466,19 @@ def output_display(result: Result) -> None: if options.null_input: # Don't read stdin, evaluate with only the activation context. try: - result = prgm.evaluate(activation) + result_value = prgm.evaluate(activation) if options.boolean: - if isinstance(result, (celtypes.BoolType, bool)): - summary = 0 if result else 1 + if isinstance(result_value, (celtypes.BoolType, bool)): + summary = 0 if result_value else 1 else: logger.warning( - "Expected celtypes.BoolType, got %s = %r", type(result), result + "Expected celtypes.BoolType, got %s = %r", + type(result_value), + result_value, ) summary = 2 else: - output_display(result) + output_display(result_value) summary = 0 except CELEvalError as ex: print( diff --git a/src/celpy/adapter.py b/src/celpy/adapter.py index a631729..154af76 100644 --- a/src/celpy/adapter.py +++ b/src/celpy/adapter.py @@ -12,15 +12,16 @@ # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and limitations under the License. + """ -Type Adapter to convert Python-native types into CEL structures. +Converts some Python-native types into CEL structures. -Currently, Atomic Python objects have direct use of types in :mod:`celpy.celtypes`. +Currently, atomic Python objects have direct use of types in :mod:`celpy.celtypes`. -Non-Atomic Python objects are characterized by JSON and Protobuf -objects. This module has functions to convert JSON objects to CEL. +Non-atomic Python objects are characterized by JSON and Protobuf messages. +This module has functions to convert JSON objects to CEL. -The protobuf decoder is TBD. +A proper protobuf decoder is TBD. A more sophisticated type injection capability may be needed to permit additional types or extensions to :mod:`celpy.celtypes`. diff --git a/src/celpy/c7nlib.py b/src/celpy/c7nlib.py index df9be92..b9120af 100644 --- a/src/celpy/c7nlib.py +++ b/src/celpy/c7nlib.py @@ -26,9 +26,9 @@ The API ======= -C7N uses CEL and the :py:mod:`c7nlib` module as follows:: +A C7N implementation can use CEL expressions and the :py:mod:`c7nlib` module as follows:: - class CELFilter(c7n.filters.core.Filter): # See below for the long list of mixins. + class CELFilter(c7n.filters.core.Filter): decls = { "resource": celpy.celtypes.MapType, "now": celpy.celtypes.TimestampType, @@ -58,17 +58,19 @@ def process(self, if self.pgm.evaluate(cel_activation): yield resource -This isn't the whole story, this is the starting point. +The :py:mod:`celpy.c7nlib` library of functions is bound into the CEL :py:class:`celpy.__init__.Runner` object that's built from the AST. -This library of functions is bound into the program that's built from the AST. +Several variables will be required in the :py:class:`celpy.evaluation.Activation` for use by most CEL expressions +that implement C7N filters: -Several objects are required in activation for use by the CEL expression +:resource: + A JSON document describing a cloud resource. -- ``resource``. The JSON document describing the cloud resource. +:now: + The current timestamp. -- ``now.`` The current timestamp. - -- Optionally, ``event`` may have an AWS CloudWatch Event. +:event: + May be needed; it should be a JSON document describing an AWS CloudWatch event. The type: value Features @@ -249,11 +251,12 @@ def process(self, resources): This would be provided in the activation context for CEL. To keep the library functions looking simple, the module global ``C7N`` is used. -This avoids introducing a non-CEL parameter to the c7nlib functions. +This avoids introducing a non-CEL parameter to the :py:mod:`celpy.c7nlib` functions. The ``C7N`` context object contains the following attributes: -- ``filter``. The original C7N ``Filter`` object. This provides access to the +:filter: + The original C7N ``Filter`` object. This provides access to the resource manager. It can be used to manage supplemental queries using C7N caches and other resource management. @@ -288,7 +291,6 @@ def process(self, resources): from types import TracebackType from typing import Any, Callable, Dict, Iterator, List, Optional, Type, Union, cast -# import dateutil from pendulum import parse as parse_date import jmespath # type: ignore [import-untyped] @@ -367,7 +369,7 @@ def key(source: celtypes.ListType, target: celtypes.StringType) -> celtypes.Valu matches: Iterator[celtypes.Value] = ( item for item in source - if cast(celtypes.StringType, cast(celtypes.MapType, item).get(key)) == target # noqa: W503 + if cast(celtypes.StringType, cast(celtypes.MapType, item).get(key)) == target ) try: return cast(celtypes.MapType, next(matches)).get(value) @@ -430,6 +432,8 @@ def __contains__(self, other): # type: ignore[no-untyped-def] return self.supernet_of(other) # type: ignore[no-untyped-call] return super(IPv4Network, self).__contains__(other) + contains = __contains__ + if sys.version_info.major == 3 and sys.version_info.minor <= 6: # pragma: no cover @staticmethod @@ -440,7 +444,7 @@ def _is_subnet_of(a, b): # type: ignore[no-untyped-def] raise TypeError(f"{a} and {b} are not of the same version") return ( b.network_address <= a.network_address - and b.broadcast_address >= a.broadcast_address # noqa: W503 + and b.broadcast_address >= a.broadcast_address ) except AttributeError: raise TypeError( @@ -684,10 +688,7 @@ def image(resource: celtypes.MapType) -> celtypes.Value: creation_date = "2000-01-01T01:01:01.000Z" image_name = "" - return json_to_cel( - # {"CreationDate": dateutil.parser.isoparse(creation_date), "Name": image_name} - {"CreationDate": parse_date(creation_date), "Name": image_name} - ) + return json_to_cel({"CreationDate": parse_date(creation_date), "Name": image_name}) def get_raw_metrics(request: celtypes.MapType) -> celtypes.Value: @@ -1462,7 +1463,7 @@ def parse_attribute_value(v: str) -> Union[int, bool, str]: results = client.describe_load_balancer_attributes( LoadBalancerArn=resource["LoadBalancerArn"] ) - print(results) + # print(results) return json_to_cel( dict( (item["Key"], parse_attribute_value(item["Value"])) @@ -1641,7 +1642,7 @@ def web_acls(resource: celtypes.MapType) -> celtypes.Value: class C7N_Interpreted_Runner(InterpretedRunner): """ - Extends the Evaluation to introduce the C7N CELFilter instance into the exvaluation. + Extends the Evaluation to introduce the C7N CELFilter instance into the evaluation. The variable is global to allow the functions to have the simple-looking argument values that CEL expects. This allows a function in this module to reach outside CEL for @@ -1655,9 +1656,8 @@ def evaluate( ) -> celtypes.Value: e = Evaluator( ast=self.ast, - activation=self.new_activation(context), - functions=self.functions, + activation=self.new_activation(), ) with C7NContext(filter=filter): - value = e.evaluate() + value = e.evaluate(context) return value diff --git a/src/celpy/celparser.py b/src/celpy/celparser.py index 4e7d356..6f04858 100644 --- a/src/celpy/celparser.py +++ b/src/celpy/celparser.py @@ -14,19 +14,19 @@ # See the License for the specific language governing permissions and limitations under the License. """ -CEL Parser. +A **Facade** around the CEL parser. -See https://github.com/google/cel-spec/blob/master/doc/langdef.md +The Parser is an instance of the :py:class:`lark.Lark` class. -https://github.com/google/cel-cpp/blob/master/parser/Cel.g4 +The grammar is in the ``cel.lark`` file. -https://github.com/google/cel-go/blob/master/parser/gen/CEL.g4 +For more information on CEL syntax, see the following: -Builds a parser from the supplied cel.lark grammar. +- https://github.com/google/cel-spec/blob/master/doc/langdef.md -.. todo:: Consider embedding the ``cel.lark`` file as a triple-quoted literal. +- https://github.com/google/cel-cpp/blob/master/parser/Cel.g4 - This means fixing a LOT of \\'s. But it also eliminates a data file from the installation. +- https://github.com/google/cel-go/blob/master/parser/gen/CEL.g4 Example:: @@ -71,6 +71,8 @@ class CELParseError(Exception): + """A syntax error in the CEL expression.""" + def __init__( self, *args: Any, line: Optional[int] = None, column: Optional[int] = None ) -> None: @@ -80,11 +82,29 @@ def __init__( class CELParser: - """Wrapper for the CEL parser and the syntax error messages.""" + """ + Creates a Lark parser with the required options. + + .. important:: **Singleton** + + There is one CEL_PARSER instance created by this class. + This is an optimization for environments like C7N where numerous + CEL expressions may parsed. + + :: + + CELParse.CEL_PARSER = None + + Is required to create another parser instance. + This is commonly required in test environments. + + This is also an **Adapter** for the CEL parser to provide pleasant + syntax error messages. + """ CEL_PARSER: Optional[Lark] = None - def __init__(self) -> None: + def __init__(self, tree_class: type = lark.Tree) -> None: if CELParser.CEL_PARSER is None: CEL_grammar = (Path(__file__).parent / "cel.lark").read_text() CELParser.CEL_PARSER = Lark( @@ -97,6 +117,7 @@ def __init__(self) -> None: propagate_positions=True, maybe_placeholders=False, priority="invert", + tree_class=tree_class, ) @staticmethod diff --git a/src/celpy/celtypes.py b/src/celpy/celtypes.py index 2d040e6..8f6bf8f 100644 --- a/src/celpy/celtypes.py +++ b/src/celpy/celtypes.py @@ -14,7 +14,7 @@ # See the License for the specific language governing permissions and limitations under the License. """ -CEL Types: wrappers on Python types to provide CEL semantics. +Provides wrappers over Python types to provide CEL semantics. This can be used by a Python module to work with CEL-friendly values and CEL results. @@ -285,18 +285,31 @@ def logical_condition(e: Value, x: Value, y: Value) -> Value: >>> logical_condition( ... BoolType(False), StringType("Not This"), StringType("that")) StringType('that') + + .. TODO:: Consider passing closures instead of Values. + + The function can evaluate e(). + If it's True, return x(). + If it's False, return y(). + Otherwise, it's a CELEvalError, which is the result """ if not isinstance(e, BoolType): raise TypeError(f"Unexpected {type(e)} ? {type(x)} : {type(y)}") - result = x if e else y - logger.debug("logical_condition(%r, %r, %r) = %r", e, x, y, result) - return result + result_value = x if e else y + logger.debug("logical_condition(%r, %r, %r) = %r", e, x, y, result_value) + return result_value def logical_and(x: Value, y: Value) -> Value: """ Native Python has a left-to-right rule. CEL && is commutative with non-Boolean values, including error objects. + + .. TODO:: Consider passing closures instead of Values. + + The function can evaluate x(). + If it's False, then return False. + Otherwise, it's True or a CELEvalError, return y(). """ if not isinstance(x, BoolType) and not isinstance(y, BoolType): raise TypeError(f"{type(x)} {x!r} and {type(y)} {y!r}") @@ -316,20 +329,23 @@ def logical_and(x: Value, y: Value) -> Value: def logical_not(x: Value) -> Value: """ - Native python `not` isn't fully exposed for CEL types. + A function for native python `not`. + + This could almost be `logical_or = evaluation.boolean(operator.not_)`, + but the definition would expose Python's notion of "truthiness", which isn't appropriate for CEL. """ if isinstance(x, BoolType): - result = BoolType(not x) + result_value = BoolType(not x) else: raise TypeError(f"not {type(x)}") - logger.debug("logical_not(%r) = %r", x, result) - return result + logger.debug("logical_not(%r) = %r", x, result_value) + return result_value def logical_or(x: Value, y: Value) -> Value: """ - Native Python has a left-to-right rule: (True or y) is True, (False or y) is y. - CEL || is commutative with non-Boolean values, including errors. + Native Python has a left-to-right rule: ``(True or y)`` is True, ``(False or y)`` is y. + CEL ``||`` is commutative with non-Boolean values, including errors. ``(x || false)`` is ``x``, and ``(false || y)`` is ``y``. Example 1:: @@ -344,7 +360,13 @@ def logical_or(x: Value, y: Value) -> Value: is a "True" - If the operand(s) are not BoolType, we'll create an TypeError that will become a CELEvalError. + If the operand(s) are not ``BoolType``, we'll create an ``TypeError`` that will become a ``CELEvalError``. + + .. TODO:: Consider passing closures instead of Values. + + The function can evaluate x(). + If it's True, then return True. + Otherwise, it's False or a CELEvalError, return y(). """ if not isinstance(x, BoolType) and not isinstance(y, BoolType): raise TypeError(f"{type(x)} {x!r} or {type(y)} {y!r}") @@ -364,9 +386,9 @@ def logical_or(x: Value, y: Value) -> Value: class BoolType(int): """ - Native Python permits unary operators on Booleans. + Native Python permits all unary operators to work on ``bool`` objects. - For CEL, We need to prevent -false from working. + For CEL, we need to prevent the CEL expression ``-false`` from working. """ def __new__(cls: Type["BoolType"], source: Any) -> "BoolType": @@ -410,7 +432,7 @@ def __new__( elif isinstance(source, MessageType): return super().__new__( cls, - cast(bytes, source.get(StringType("value"))), # type: ignore [attr-defined] + cast(bytes, source.get(StringType("value"))), ) elif isinstance(source, Iterable): return super().__new__(cls, cast(Iterable[int], source)) @@ -490,9 +512,9 @@ def int64(operator: IntOperator) -> IntOperator: @wraps(operator) def clamped_operator(*args: Any, **kwargs: Any) -> int: - result: int = operator(*args, **kwargs) - if -(2**63) <= result < 2**63: - return result + result_value: int = operator(*args, **kwargs) + if -(2**63) <= result_value < 2**63: + return result_value raise ValueError("overflow") return cast(IntOperator, clamped_operator) @@ -650,9 +672,9 @@ def uint64(operator: IntOperator) -> IntOperator: @wraps(operator) def clamped_operator(*args: Any, **kwargs: Any) -> int: - result = operator(*args, **kwargs) - if 0 <= result < 2**64: - return result + result_value = operator(*args, **kwargs) + if 0 <= result_value < 2**64: + return result_value raise ValueError("overflow") return cast(IntOperator, clamped_operator) @@ -825,14 +847,14 @@ def equal(s: Any, o: Any) -> Value: except TypeError as ex: return cast(BoolType, ex) # Instead of Union[BoolType, TypeError] - result = len(self) == len(other) and reduce( # noqa: W503 + result_value = len(self) == len(other) and reduce( logical_and, # type: ignore [arg-type] (equal(item_s, item_o) for item_s, item_o in zip(self, other)), BoolType(True), # type: ignore [arg-type] ) - if isinstance(result, TypeError): - raise result - return bool(result) + if isinstance(result_value, TypeError): + raise result_value + return bool(result_value) def __ne__(self, other: Any) -> bool: if not isinstance(other, (list, ListType)): @@ -844,14 +866,17 @@ def not_equal(s: Any, o: Any) -> Value: except TypeError as ex: return cast(BoolType, ex) # Instead of Union[BoolType, TypeError] - result = len(self) != len(other) or reduce( # noqa: W503 + result_value = len(self) != len(other) or reduce( logical_or, # type: ignore [arg-type] (not_equal(item_s, item_o) for item_s, item_o in zip(self, other)), BoolType(False), # type: ignore [arg-type] ) - if isinstance(result, TypeError): - raise result - return bool(result) + if isinstance(result_value, TypeError): + raise result_value + return bool(result_value) + + def contains(self, item: Value) -> BoolType: + return BoolType(item in self) BaseMapTypes = Union[Mapping[Any, Any], Sequence[Tuple[Any, Any]], None] @@ -880,7 +905,10 @@ def __init__(self, items: BaseMapTypes = None) -> None: if items is None: pass elif isinstance(items, Sequence): + # Must Be Unique for name, value in items: + if name in self: + raise ValueError(f"Duplicate key {name!r}") self[name] = value elif isinstance(items, Mapping): for name, value in items.items(): @@ -908,14 +936,14 @@ def equal(s: Any, o: Any) -> BoolType: keys_s = self.keys() keys_o = other.keys() - result = keys_s == keys_o and reduce( # noqa: W503 + result_value = keys_s == keys_o and reduce( logical_and, # type: ignore [arg-type] (equal(self[k], other[k]) for k in keys_s), BoolType(True), # type: ignore [arg-type] ) - if isinstance(result, TypeError): - raise result - return bool(result) + if isinstance(result_value, TypeError): + raise result_value + return bool(result_value) def __ne__(self, other: Any) -> bool: if not isinstance(other, (Mapping, MapType)): @@ -936,20 +964,34 @@ def not_equal(s: Any, o: Any) -> BoolType: keys_s = self.keys() keys_o = other.keys() - result = keys_s != keys_o or reduce( # noqa: W503 + result_value = keys_s != keys_o or reduce( logical_or, # type: ignore [arg-type] (not_equal(self[k], other[k]) for k in keys_s), BoolType(False), # type: ignore [arg-type] ) - if isinstance(result, TypeError): - raise result - return bool(result) + if isinstance(result_value, TypeError): + raise result_value + return bool(result_value) + + def get(self, key: Any, default: Optional[Any] = None) -> Value: + """There is no default provision in CEL, that's a Python feature.""" + if not MapType.valid_key_type(key): + raise TypeError(f"unsupported key type: {type(key)}") + if key in self: + return super().get(key) + elif default is not None: + return cast(Value, default) + else: + raise KeyError(key) @staticmethod def valid_key_type(key: Any) -> bool: """Valid CEL key types. Plus native str for tokens in the source when evaluating ``e.f``""" return isinstance(key, (IntType, UintType, BoolType, StringType, str)) + def contains(self, item: Value) -> BoolType: + return BoolType(item in self) + class NullType: """Python's None semantics aren't quite right for CEL.""" @@ -993,6 +1035,9 @@ def __ne__(self, other: Any) -> bool: def __hash__(self) -> int: return super().__hash__() + def contains(self, item: Value) -> BoolType: + return BoolType(cast(StringType, item) in self) + class TimestampType(datetime.datetime): """ @@ -1079,7 +1124,7 @@ def __new__( return ts elif isinstance(source, str): - # Use dateutil to try a variety of text formats. + # Use ``pendulum`` to try a variety of text formats. parsed_datetime = cast(datetime.datetime, pendulum.parse(source)) return super().__new__( cls, @@ -1107,17 +1152,17 @@ def __str__(self) -> str: def __add__(self, other: Any) -> "TimestampType": """Timestamp + Duration -> Timestamp""" - result = super().__add__(other) - if result == NotImplemented: + result_value = super().__add__(other) + if result_value == NotImplemented: return NotImplemented - return TimestampType(result) + return TimestampType(result_value) def __radd__(self, other: Any) -> "TimestampType": """Duration + Timestamp -> Timestamp""" - result = super().__radd__(other) - if result == NotImplemented: + result_value = super().__radd__(other) + if result_value == NotImplemented: return NotImplemented - return TimestampType(result) + return TimestampType(result_value) # For more information, check the typeshed definition # https://github.com/python/typeshed/blob/master/stdlib/2and3/datetime.pyi @@ -1131,17 +1176,17 @@ def __sub__(self, other: "DurationType") -> "TimestampType": ... # pragma: no c def __sub__( self, other: Union["TimestampType", "DurationType"] ) -> Union["TimestampType", "DurationType"]: - result = super().__sub__(other) - if result == NotImplemented: - return cast(DurationType, result) - if isinstance(result, datetime.timedelta): - return DurationType(result) - return TimestampType(result) + result_value = super().__sub__(other) + if result_value == NotImplemented: + return cast(DurationType, result_value) + if isinstance(result_value, datetime.timedelta): + return DurationType(result_value) + return TimestampType(result_value) @classmethod def tz_name_lookup(cls, tz_name: str) -> Optional[datetime.tzinfo]: """ - The :py:func:`dateutil.tz.gettz` may be extended with additional aliases. + The ``pendulum`` parsing may be extended with additional aliases. .. TODO: Permit an extension into the timezone lookup. Tweak ``celpy.celtypes.TimestampType.TZ_ALIASES``. @@ -1332,13 +1377,13 @@ def __add__(self, other: Any) -> "DurationType": A duration + timestamp is not implemented by the timedelta superclass; it is handled by the datetime superclass that implementes timestamp + duration. """ - result = super().__add__(other) - if result == NotImplemented: - return cast(DurationType, result) + result_value = super().__add__(other) + if result_value == NotImplemented: + return cast(DurationType, result_value) # This is handled by TimestampType; this is here for completeness, but isn't used. - if isinstance(result, (datetime.datetime, TimestampType)): - return TimestampType(result) # pragma: no cover - return DurationType(result) + if isinstance(result_value, (datetime.datetime, TimestampType)): + return TimestampType(result_value) # pragma: no cover + return DurationType(result_value) def __radd__(self, other: Any) -> "DurationType": # pragma: no cover """ @@ -1346,13 +1391,13 @@ def __radd__(self, other: Any) -> "DurationType": # pragma: no cover Most cases are handled by TimeStamp. """ - result = super().__radd__(other) - if result == NotImplemented: - return cast(DurationType, result) + result_value = super().__radd__(other) + if result_value == NotImplemented: + return cast(DurationType, result_value) # This is handled by TimestampType; this is here for completeness, but isn't used. - if isinstance(result, (datetime.datetime, TimestampType)): - return TimestampType(result) - return DurationType(result) + if isinstance(result_value, (datetime.datetime, TimestampType)): + return TimestampType(result_value) + return DurationType(result_value) def getHours(self, tz_name: Optional[str] = None) -> IntType: assert tz_name is None @@ -1422,83 +1467,14 @@ def __init__(self, *args: Value, **fields: Value) -> None: else: super().__init__({StringType(k): v for k, v in fields.items()}) - # def get(self, field: Any, default: Optional[Value] = None) -> Value: - # """ - # Alternative implementation with descent to locate a deeply-buried field. - # It seemed like this was the defined behavior. It turns it, it isn't. - # The code is here in case we're wrong and it really is the defined behavior. - # - # Note. There is no default provision in CEL. - # """ - # if field in self: - # return super().get(field) - # - # def descend(message: MessageType, field: Value) -> MessageType: - # if field in message: - # return message - # for k in message.keys(): - # found = descend(message[k], field) - # if found is not None: - # return found - # return None - # - # sub_message = descend(self, field) - # if sub_message is None: - # return default - # return sub_message.get(field) - - -class TypeType: + +class TypeType(type): """ - Annotation used to mark protobuf type objects. - We map these to CELTypes so that type name testing works. + This is primarily used as a function to extract type from an object or a type. + For consistence, we define it as a type so other types can extend it. """ - type_name_mapping = { - "google.protobuf.Duration": DurationType, - "google.protobuf.Timestamp": TimestampType, - "google.protobuf.Int32Value": IntType, - "google.protobuf.Int64Value": IntType, - "google.protobuf.UInt32Value": UintType, - "google.protobuf.UInt64Value": UintType, - "google.protobuf.FloatValue": DoubleType, - "google.protobuf.DoubleValue": DoubleType, - "google.protobuf.Value": MessageType, - "google.protubuf.Any": MessageType, # Weird. - "google.protobuf.Any": MessageType, - "list_type": ListType, - "map_type": MapType, - "map": MapType, - "list": ListType, - "string": StringType, - "bytes": BytesType, - "bool": BoolType, - "int": IntType, - "uint": UintType, - "double": DoubleType, - "null_type": type(None), - "STRING": StringType, - "BOOL": BoolType, - "INT64": IntType, - "UINT64": UintType, - "INT32": IntType, - "UINT32": UintType, - "BYTES": BytesType, - "DOUBLE": DoubleType, - } - - def __init__(self, value: Any = "") -> None: - if isinstance(value, str) and value in self.type_name_mapping: - self.type_reference = self.type_name_mapping[value] - elif isinstance(value, str): - try: - self.type_reference = eval(value) - except (NameError, SyntaxError): - raise TypeError(f"Unknown type {value!r}") - else: - self.type_reference = value.__class__ - - def __eq__(self, other: Any) -> bool: - return ( - other == self.type_reference or isinstance(other, self.type_reference) # noqa: W503 - ) + def __new__(typ, instance: Any) -> type: + if type(instance) is type: + return typ + return type(instance) diff --git a/src/celpy/evaluation.py b/src/celpy/evaluation.py index d9d40ff..2a32ef5 100644 --- a/src/celpy/evaluation.py +++ b/src/celpy/evaluation.py @@ -14,30 +14,38 @@ # See the License for the specific language governing permissions and limitations under the License. """ -CEL Interpreter using the AST directly. +Evaluates CEL expressions given an AST. + +There are two implementations: + +- Evaluator -- interprets the AST directly. + +- Transpiler -- transpiles the AST to Python, compiles the Python to create a code object, and then uses :py:func:`exec` to evaluate the code object. The general idea is to map CEL operators to Python operators and push the real work off to Python objects defined by the :py:mod:`celpy.celtypes` module. -CEL operator "+" is implemented by "_+_" function. We map this to :py:func:`operator.add`. -This will then look for `__add__()` methods in the various :py:class:`celpy.celtypes.CELType` +CEL operator ``+`` is implemented by a ``"_+_"`` function. +We map this name to :py:func:`operator.add`. +This will then look for :py:meth:`__add__` methods in the various :py:mod:`celpy.celtypes` types. In order to deal gracefully with missing and incomplete data, -exceptions are turned into first-class :py:class:`Result` objects. +checked exceptions are used. +A raised exception is turned into first-class :py:class:`celpy.celtypes.Result` object. They're not raised directly, but instead saved as part of the evaluation so that short-circuit operators can ignore the exceptions. This means that Python exceptions like :exc:`TypeError`, :exc:`IndexError`, and :exc:`KeyError` are caught and transformed into :exc:`CELEvalError` objects. -The :py:class:`Resut` type hint is a union of the various values that are encountered +The :py:class:`celpy.celtypes.Result` type hint is a union of the various values that are encountered during evaluation. It's a union of the :py:class:`celpy.celtypes.CELTypes` type and the :exc:`CELEvalError` exception. .. important:: Debugging - If the os environment variable ``CEL_TRACE`` is set, then detailed tracing of methods is made available. + If the OS environment variable :envvar:`CEL_TRACE` is set, then detailed tracing of methods is made available. To see the trace, set the logging level for ``celpy.Evaluator`` to ``logging.DEBUG``. """ @@ -47,8 +55,10 @@ import operator import os import re +from string import Template import sys from functools import reduce, wraps +from textwrap import dedent from typing import ( Any, Callable, @@ -99,9 +109,9 @@ def function_matches(text: str, pattern: str) -> "Result": return celpy.celtypes.BoolType(m is not None) -# This annotation describes a union of types, functions, and function types. +# An Annotation describes a union of types, functions, and function types. Annotation = Union[ - celpy.celtypes.CELType, + celpy.celtypes.TypeType, Callable[ ..., celpy.celtypes.Value ], # Conversion functions and protobuf message type @@ -225,7 +235,7 @@ def __call__(self, *args: Any) -> "CELEvalError": return self -# The interim results extend ``celtypes`` to include itermediate ``CELEvalError`` exception objects. +# The interim results extend ``celtypes`` to include intermediate ``CELEvalError`` exception objects. # These can be deferred as part of commutative logical_and and logical_or operations. # It includes the responses to ``type()`` queries, also. Result = Union[ @@ -257,7 +267,7 @@ def eval_error( Wrap a function to transform native Python exceptions to CEL CELEvalError values. Any exception of the given class is replaced with the new CELEvalError object. - :param new_text: Text of the exception, e.g., "divide by zero", "no such overload") + :param new_text: Text of the exception, e.g., "divide by zero", "no such overload", this is the return value if the :exc:`CELEvalError` becomes the result. :param exc_class: A Python exception class to match, e.g. ZeroDivisionError, or a sequence of exception classes (e.g. (ZeroDivisionError, ValueError)) @@ -295,7 +305,7 @@ def boolean( function: Callable[..., celpy.celtypes.Value], ) -> Callable[..., celpy.celtypes.BoolType]: """ - Wraps boolean operators to create CEL BoolType results. + Wraps operators to create CEL BoolType results. :param function: One of the operator.lt, operator.gt, etc. comparison functions :return: Decorated function with type coercion. @@ -305,10 +315,10 @@ def boolean( def bool_function( a: celpy.celtypes.Value, b: celpy.celtypes.Value ) -> celpy.celtypes.BoolType: - result = function(a, b) - if result == NotImplemented: - return cast(celpy.celtypes.BoolType, result) - return celpy.celtypes.BoolType(bool(result)) + result_value = function(a, b) + if result_value == NotImplemented: + return cast(celpy.celtypes.BoolType, result_value) + return celpy.celtypes.BoolType(bool(result_value)) return bool_function @@ -333,40 +343,41 @@ def operator_in(item: Result, container: Result) -> Result: would do it. But. It's not quite right for the job. There need to be three results, something :py:func:`filter` doesn't handle. - These are the chocies: + These are the choices: - True. There was a item found. Exceptions may or may not have been found. - - False. No item found AND no expceptions. + - False. No item found AND no exceptions. - CELEvalError. No item found AND at least one exception. To an extent this is a little like the ``exists()`` macro. We can think of ``container.contains(item)`` as ``container.exists(r, r == item)``. - However, exists() tends to silence exceptions, where this can expost them. + However, exists() tends to silence exceptions, where this can expose them. .. todo:: This may be better done as ``reduce(logical_or, (item == c for c in container), BoolType(False))`` """ - result: Result = celpy.celtypes.BoolType(False) + result_value: Result = celpy.celtypes.BoolType(False) for c in cast(Iterable[Result], container): try: if c == item: return celpy.celtypes.BoolType(True) except TypeError as ex: logger.debug("operator_in(%s, %s) --> %s", item, container, ex) - result = CELEvalError("no such overload", ex.__class__, ex.args) - logger.debug("operator_in(%r, %r) = %r", item, container, result) - return result + result_value = CELEvalError("no such overload", ex.__class__, ex.args) + logger.debug("operator_in(%r, %r) = %r", item, container, result_value) + return result_value def function_size(container: Result) -> Result: """ - The size() function applied to a Value. Delegate to Python's :py:func:`len`. + The size() function applied to a Value. + This is delegated to Python's :py:func:`len`. - (string) -> int string length - (bytes) -> int bytes length - (list(A)) -> int list size - (map(A, B)) -> int map size + size(string) -> int string length + size(bytes) -> int bytes length + size(list(A)) -> int list size + size(map(A, B)) -> int map size For other types, this will raise a Python :exc:`TypeError`. (This is captured and becomes an :exc:`CELEvalError` Result.) @@ -377,13 +388,132 @@ def function_size(container: Result) -> Result: if container is None: return celpy.celtypes.IntType(0) sized_container = cast(Sized, container) - result = celpy.celtypes.IntType(len(sized_container)) - logger.debug("function_size(%r) = %r", container, result) - return result + result_value = celpy.celtypes.IntType(len(sized_container)) + logger.debug("function_size(%r) = %r", container, result_value) + return result_value + + +def function_contains( + container: Union[ + celpy.celtypes.ListType, celpy.celtypes.MapType, celpy.celtypes.StringType + ], + item: Result, +) -> Result: + """ + The contains() function applied to a Container and a Value. + THis is delegated to the `contains` method of a class. + """ + return celpy.celtypes.BoolType(container.contains(cast(celpy.celtypes.Value, item))) + + +def function_startsWith( + string: celpy.celtypes.StringType, fragment: celpy.celtypes.StringType +) -> Result: + return celpy.celtypes.BoolType(string.startswith(fragment)) + + +def function_endsWith( + string: celpy.celtypes.StringType, fragment: celpy.celtypes.StringType +) -> Result: + return celpy.celtypes.BoolType(string.endswith(fragment)) + + +def function_getDate( + ts: celpy.celtypes.TimestampType, + tz_name: Optional[celpy.celtypes.StringType] = None, +) -> Result: + return celpy.celtypes.IntType(ts.getDate(tz_name)) + + +def function_getDayOfMonth( + ts: celpy.celtypes.TimestampType, + tz_name: Optional[celpy.celtypes.StringType] = None, +) -> Result: + return celpy.celtypes.IntType(ts.getDayOfMonth(tz_name)) + + +def function_getDayOfWeek( + ts: celpy.celtypes.TimestampType, + tz_name: Optional[celpy.celtypes.StringType] = None, +) -> Result: + return celpy.celtypes.IntType(ts.getDayOfWeek(tz_name)) + + +def function_getDayOfYear( + ts: celpy.celtypes.TimestampType, + tz_name: Optional[celpy.celtypes.StringType] = None, +) -> Result: + return celpy.celtypes.IntType(ts.getDayOfYear(tz_name)) + + +def function_getFullYear( + ts: celpy.celtypes.TimestampType, + tz_name: Optional[celpy.celtypes.StringType] = None, +) -> Result: + return celpy.celtypes.IntType(ts.getFullYear(tz_name)) + + +def function_getMonth( + ts: celpy.celtypes.TimestampType, + tz_name: Optional[celpy.celtypes.StringType] = None, +) -> Result: + return celpy.celtypes.IntType(ts.getMonth(tz_name)) + + +def function_getHours( + ts: celpy.celtypes.TimestampType, + tz_name: Optional[celpy.celtypes.StringType] = None, +) -> Result: + return celpy.celtypes.IntType(ts.getHours(tz_name)) + + +def function_getMilliseconds( + ts: celpy.celtypes.TimestampType, + tz_name: Optional[celpy.celtypes.StringType] = None, +) -> Result: + return celpy.celtypes.IntType(ts.getMilliseconds(tz_name)) + + +def function_getMinutes( + ts: celpy.celtypes.TimestampType, + tz_name: Optional[celpy.celtypes.StringType] = None, +) -> Result: + return celpy.celtypes.IntType(ts.getMinutes(tz_name)) + + +def function_getSeconds( + ts: celpy.celtypes.TimestampType, + tz_name: Optional[celpy.celtypes.StringType] = None, +) -> Result: + return celpy.celtypes.IntType(ts.getSeconds(tz_name)) + + +def bool_lt(a: Result, b: Result) -> Result: + return boolean(operator.lt)(a, b) + + +def bool_le(a: Result, b: Result) -> Result: + return boolean(operator.le)(a, b) + + +def bool_gt(a: Result, b: Result) -> Result: + return boolean(operator.gt)(a, b) + + +def bool_ge(a: Result, b: Result) -> Result: + return boolean(operator.ge)(a, b) + + +def bool_eq(a: Result, b: Result) -> Result: + return boolean(operator.eq)(a, b) + + +def bool_ne(a: Result, b: Result) -> Result: + return boolean(operator.ne)(a, b) # User-defined functions can override items in this mapping. -base_functions: Mapping[str, CELFunction] = { +base_functions: dict[str, CELFunction] = { "!_": celpy.celtypes.logical_not, "-_": operator.neg, "_+_": operator.add, @@ -391,49 +521,42 @@ def function_size(container: Result) -> Result: "_*_": operator.mul, "_/_": operator.truediv, "_%_": operator.mod, - "_<_": boolean(operator.lt), - "_<=_": boolean(operator.le), - "_>=_": boolean(operator.ge), - "_>_": boolean(operator.gt), - "_==_": boolean(operator.eq), - "_!=_": boolean(operator.ne), + "_<_": bool_lt, + "_<=_": bool_le, + "_>_": bool_gt, + "_>=_": bool_ge, + "_==_": bool_eq, + "_!=_": bool_ne, "_in_": operator_in, "_||_": celpy.celtypes.logical_or, "_&&_": celpy.celtypes.logical_and, "_?_:_": celpy.celtypes.logical_condition, "_[_]": operator.getitem, + # The "methods" are actually named functions that can be overridden. + # The function version delegates to class methods. + # Yes, it's a bunch of indirection, but it permits simple overrides. + # A number of types support "size" and "contains": StringType, MapType, ListType + # This is generally made available via the _in_ operator. "size": function_size, - # StringType methods - "endsWith": lambda s, text: celpy.celtypes.BoolType(s.endswith(text)), - "startsWith": lambda s, text: celpy.celtypes.BoolType(s.startswith(text)), + "contains": function_contains, + # Universally available + "type": celpy.celtypes.TypeType, + # StringType methods, used by :py:meth:`Evaluator.method_eval` + "endsWith": function_endsWith, + "startsWith": function_startsWith, "matches": function_matches, - "contains": lambda s, text: celpy.celtypes.BoolType(text in s), # TimestampType methods. Type details are redundant, but required because of the lambdas - "getDate": lambda ts, tz_name=None: celpy.celtypes.IntType(ts.getDate(tz_name)), - "getDayOfMonth": lambda ts, tz_name=None: celpy.celtypes.IntType( - ts.getDayOfMonth(tz_name) - ), - "getDayOfWeek": lambda ts, tz_name=None: celpy.celtypes.IntType( - ts.getDayOfWeek(tz_name) - ), - "getDayOfYear": lambda ts, tz_name=None: celpy.celtypes.IntType( - ts.getDayOfYear(tz_name) - ), - "getFullYear": lambda ts, tz_name=None: celpy.celtypes.IntType( - ts.getFullYear(tz_name) - ), - "getMonth": lambda ts, tz_name=None: celpy.celtypes.IntType(ts.getMonth(tz_name)), + "getDate": function_getDate, + "getDayOfMonth": function_getDayOfMonth, + "getDayOfWeek": function_getDayOfWeek, + "getDayOfYear": function_getDayOfYear, + "getFullYear": function_getFullYear, + "getMonth": function_getMonth, # TimestampType and DurationType methods - "getHours": lambda ts, tz_name=None: celpy.celtypes.IntType(ts.getHours(tz_name)), - "getMilliseconds": lambda ts, tz_name=None: celpy.celtypes.IntType( - ts.getMilliseconds(tz_name) - ), - "getMinutes": lambda ts, tz_name=None: celpy.celtypes.IntType( - ts.getMinutes(tz_name) - ), - "getSeconds": lambda ts, tz_name=None: celpy.celtypes.IntType( - ts.getSeconds(tz_name) - ), + "getHours": function_getHours, + "getMilliseconds": function_getMilliseconds, + "getMinutes": function_getMinutes, + "getSeconds": function_getSeconds, # type conversion functions "bool": celpy.celtypes.BoolType, "bytes": celpy.celtypes.BytesType, @@ -446,7 +569,6 @@ def function_size(container: Result) -> Result: "string": celpy.celtypes.StringType, "timestamp": celpy.celtypes.TimestampType, "uint": celpy.celtypes.UintType, - "type": type, } @@ -454,29 +576,30 @@ class Referent: """ A Name can refer to any of the following things: - - Annotations -- initially most names are these - or a CELFunction that may implement a type. + - ``Annotation`` -- initially most names are these. Must be provided as part of the initialization. - - NameContainer -- some names are these. This is true - when the name is *not* provided as part of the initialization because + - ``CELFunction`` -- a Python function to implement a CEL function or method. + Must be provided as part of the initialization. + The type conversion functions are names in a ``NameContainer``. + + - ``NameContainer`` -- some names are these. + This is true when the name is *not* provided as part of the initialization because we discovered the name during type or environment binding. - - celpy.celtypes.Value -- many annotations also have values. + - ``celpy.celtypes.Value`` -- many annotations also have values. These are provided **after** Annotations, and require them. - - CELEvalError -- This seems unlikely, but we include it because it's possible. + - ``CELEvalError`` -- This seems unlikely, but we include it because it's possible. - - Functions -- All of the type conversion functions are names in a NameContainer. - - A name can be ambiguous and refer to both a nested ``NameContainer`` as well - as a ``celpy.celtypes.Value`` (usually a MapType instance.) + A name can be ambiguous and refer to a nested ``NameContainer`` as well + as a ``celpy.celtypes.Value`` (usually a ``MapType`` instance.) Object ``b`` has two possible meanings: - - ``b.c`` is a NameContainer for ``c``, a string. + - ``b`` is a ``NameContainer`` with ``c``, a string or some other object. - - ``b`` is a mapping, and ``b.c`` is syntax sugar for ``b['c']``. + - ``b`` is a ``MapType`` or ``MessageType``, and ``b.c`` is syntax sugar for ``b['c']``. The "longest name" rule means that the useful value is the "c" object in the nested ``NameContainer``. @@ -491,24 +614,23 @@ class Referent: >>> b.value == nc True - In effect, this class is - :: + .. note:: Future Design - Referent = Union[ - Annotation, - celpy.celtypes.Value, - CELEvalError, - CELFunction, - ] + A ``Referent`` is (almost) a ``tuple[Annotation, NameContainer | None, Value | NotSetSentinel]``. + The current implementation is stateful, because values are optional and may be added later. + The use of a special sentinel to indicate the value was not set is a little akward. + It's not really a 3-tuple, because NameContainers don't have values; they are a kind of value. + (``None`` is a valid value, and can't be used for this.) + + It may be slightly simpler to use a union of two types: + ``tuple[Annotation] | tuple[Annotation, NameContainer | Value]``. + One-tuples capture the Annotation for a name; two-tuples capture Annotation and Value (or subsidiary NameContainer). """ def __init__( self, ref_to: Optional[Annotation] = None, - # Union[ - # None, Annotation, celpy.celtypes.Value, CELEvalError, - # CELFunction, 'NameContainer' - # ] = None + # TODO: Add value here, also, as a handy short-cut to avoid the value setter. ) -> None: self.annotation: Optional[Annotation] = None self.container: Optional["NameContainer"] = None @@ -520,7 +642,7 @@ def __init__( CELFunction, "NameContainer", ] = None - self._value_set = False + self._value_set = False # Should NOT be private. if ref_to: self.annotation = ref_to @@ -531,6 +653,18 @@ def __repr__(self) -> str: f"_value={self._value!r})" ) + def __eq__(self, other: Any) -> bool: + # TODO: When minimum version >= 3.10, use match statement + if isinstance(other, type(self)): + same = ( + self.annotation == other.annotation + and self.container == other.container + and self._value_set == other._value_set + and (self._value == other._value if self._value_set else True) + ) + return same + return NotImplemented # pragma: no cover + @property def value( self, @@ -569,9 +703,9 @@ def clone(self) -> "Referent": return new -# A name resolution context is a mapping from an identifer to a Value or a ``NameContainer``. +# A name resolution context is a mapping from an identifier to a Value or a ``NameContainer``. # This reflects some murkiness in the name resolution algorithm that needs to be cleaned up. -Context = Mapping[str, Union[Result, "NameContainer"]] +Context = Mapping[str, Union[Result, "NameContainer", "CELFunction"]] # Copied from cel.lark @@ -617,33 +751,33 @@ class NameContainer(Dict[str, Referent]): The annotations (i.e. protobuf message types) have only a fairly remote annotation without a value. - Structure. + .. rubric:: Structure - A NameContainer is a mapping from names to Referents. + A ``NameContainer`` is a mapping from names to ``Referent`` instances. - A Referent can be one of three things. + A `Referent` can be one of several things, including... - A NameContainer further down the path - An Annotation - - An Annotation with a value. + - An Annotation and a value + - A CELFunction (In effect, an Annotation of CELFunction, and a value of the function implementation.) - Loading Names. + .. rubric:: Life and Content - There are several "phases" to building the chain of ``NameContainer`` instances. + There are two phases to building the chain of ``NameContainer`` instances. 1. The ``Activation`` creates the initial ``name : annotation`` bindings. Generally, the names are type names, like "int", bound to :py:class:`celtypes.IntType`. In some cases, the name is a future variable name, "resource", bound to :py:class:`celtypes.MapType`. - 2. The ``Activation`` creates a second ``NameContainer`` that has variable names. - This has a reference back to the parent to resolve names that are types. + 2. The ``Activation`` updates some variables to provide values. - This involves decomposing the paths of names to make a tree of nested ``NameContainers``. + A name is decomposed into a path to make a tree of nested ``NameContainers``. Upper-level containers don't (necessarily) have types or values -- they're merely ``NameContainer`` along the path to the target names. - Resolving Names. + .. rubric:: Resolving Names See https://github.com/google/cel-spec/blob/master/doc/langdef.md#name-resolution @@ -688,7 +822,7 @@ class NameContainer(Dict[str, Referent]): The evaluator's :meth:`ident_value` method resolves the identifier into the ``Referent``. - Acceptance Test Case + .. rubric:: Acceptance Test Cases We have two names @@ -741,7 +875,7 @@ def load_annotations( :param names: A dictionary of {"name1.name1....": Referent, ...} items. """ for name, refers_to in names.items(): - self.logger.debug("load_annotations %r : %r", name, refers_to) + # self.logger.debug("load_annotations %r : %r", name, refers_to) if not self.extended_name_path.match(name): raise ValueError(f"Invalid name {name}") @@ -757,9 +891,9 @@ def load_annotations( context.setdefault(final, Referent(refers_to)) def load_values(self, values: Context) -> None: - """Update annotations with actual values.""" + """Update any annotations with actual values.""" for name, refers_to in values.items(): - self.logger.debug("load_values %r : %r", name, refers_to) + # self.logger.debug("load_values %r : %r", name, refers_to) if not self.extended_name_path.match(name): raise ValueError(f"Invalid name {name}") @@ -773,7 +907,7 @@ def load_values(self, values: Context) -> None: if ref.container is None: ref.container = NameContainer(parent=self.parent) context = ref.container - context.setdefault(final, Referent()) # No annotation. + context.setdefault(final, Referent()) # No annotation previously present. context[final].value = refers_to class NotFound(Exception): @@ -785,61 +919,102 @@ class NotFound(Exception): pass @staticmethod - def dict_find_name(some_dict: Dict[str, Referent], path: List[str]) -> Result: + def dict_find_name( + some_dict: Union[Dict[str, Referent], Referent], path: Sequence[str] + ) -> Referent: """ - Extension to navgiate into mappings, messages, and packages. + Recursive navigation into mappings, messages, and packages. + These are not NameContainers (or Activations). - :param some_dict: An instance of a MapType, MessageType, or PackageType. - :param path: names to follow into the structure. + :param some_dict: An instance of a ``MapType``, ``MessageType``, or ``PackageType``. + :param path: sequence of names to follow into the structure. :returns: Value found down inside the structure. """ if path: head, *tail = path try: return NameContainer.dict_find_name( - cast(Dict[str, Referent], some_dict[head]), tail + cast(Dict[str, Referent], some_dict)[head], tail ) except KeyError: - NameContainer.logger.debug("%r not found in %s", head, some_dict.keys()) + NameContainer.logger.debug( + "%r not found in %s", + head, + cast(Dict[str, Referent], some_dict).keys(), + ) raise NameContainer.NotFound(path) else: - return cast(Result, some_dict) + # End of the path, we found it. + if isinstance(some_dict, Referent): # pragma: no cover + # Seems unlikely, but, just to be sure... + return some_dict + referent = Referent(celpy.celtypes.MapType) + referent.value = cast(celpy.celtypes.MapType, some_dict) + return referent - def find_name(self, path: List[str]) -> Union["NameContainer", Result]: + def find_name(self, path: List[str]) -> Referent: """ Find the name by searching down through nested packages or raise NotFound. + Returns the Value associated with this Name. + This is a kind of in-order tree walk of contained packages. + + The collaborator must choose the annotation or the value from the Referent. + + .. todo:: Refactored to return Referent. + + The collaborator must handle two distinct errors: + + 1. ``self[head]`` has a ``KeyError`` exception -- while not found on this path, the collaborator should keep searching. + Eventually it will raise a final ``KeyError`` that maps to a ``CELEvalError`` + This should be exposed as f"no such member in mapping: {ex.args[0]!r}" + + 2. ``self[head].value`` has no value and the ``Referent`` returned the annotation instead of the value. + In transpiled Python code, this **should** be exposed as f"undeclared reference to {ex.args[0]!r} (in container {ex.args[1]!r})" """ - if path: - head, *tail = path - try: - sub_context = self[head].value - except KeyError: - self.logger.debug("%r not found in %s", head, self.keys()) - raise NameContainer.NotFound(path) - if isinstance(sub_context, NameContainer): - return sub_context.find_name(tail) - elif isinstance( - sub_context, - ( - celpy.celtypes.MessageType, - celpy.celtypes.MapType, - celpy.celtypes.PackageType, - dict, - ), - ): - # Out of defined NameContainers, moving into Values: Messages, Mappings or Packages - # Make a fake Referent return value. - item: Union["NameContainer", Result] = NameContainer.dict_find_name( - cast(Dict[str, Referent], sub_context), tail - ) - return item - else: - # Fully matched. No more Referents with NameContainers or Referents with Mappings. - return cast(NameContainer, sub_context) + if not path: + # Already fully matched. This ``NameContainer`` is what they were looking for. + referent = Referent() + referent.value = self + return referent + + # Find the head of the path. + head, *tail = path + try: + sub_context = self[head] + except KeyError: + self.logger.debug("%r not found in %r", head, list(self.keys())) + raise NameContainer.NotFound(path) + if not tail: + # Found what they were looking for + return sub_context + + # There are several special cases for the continued search. + self.logger.debug("%r %r %r", head, tail, sub_context) + + # We found a NameContainer, simple recursion will do. + item: Referent + if sub_context.container: # isinstance(sub_context, NameContainer): + return sub_context.container.find_name(tail) + + # Uncommon case: value with no annotation, and the value is a Message, Mapping, or Package + elif sub_context._value_set and isinstance( + sub_context.value, + ( + celpy.celtypes.MessageType, + celpy.celtypes.MapType, + celpy.celtypes.PackageType, + dict, + ), + ): + item = NameContainer.dict_find_name( + cast(Dict[str, Referent], sub_context.value), tail + ) + return item + + # A primitive type, but not at the end of the path. Ugn. else: - # Fully matched. This NameContainer is what we were looking for. - return self + raise TypeError(f"{sub_context!r} not a container") def parent_iter(self) -> Iterator["NameContainer"]: """Yield this NameContainer and all of its parents to create a flat list.""" @@ -855,11 +1030,11 @@ def resolve_name(self, package: Optional[str], name: str) -> Referent: If a.b is a name to be resolved in the context of a protobuf declaration with scope A.B, then resolution is attempted, in order, as - 1. A.B.a.b. (Search for "a" in paackage "A.B"; the ".b" is handled separately.) + 1. A.B.a.b. (Search for "a" in package "A.B"; the ".b" is handled separately.) - 2. A.a.b. (Search for "a" in paackage "A"; the ".b" is handled separately.) + 2. A.a.b. (Search for "a" in package "A"; the ".b" is handled separately.) - 3. (finally) a.b. (Search for "a" in paackage None; the ".b" is handled separately.) + 3. (finally) a.b. (Search for "a" in package None; the ".b" is handled separately.) To override this behavior, one can use .a.b; this name will only be attempted to be resolved in the root scope, i.e. as a.b. @@ -869,22 +1044,23 @@ def resolve_name(self, package: Optional[str], name: str) -> Referent: Given a target, search through this ``NameContainer`` and all parents in the :meth:`parent_iter` iterable. The first name we find in the parent sequence is the goal. - This is because values are first, type annotations are laast. + This is because values are first, type annotations are last. If we can't find the identifier with given package target, truncate the package name from the end to create a new target and try again. This is a bottom-up look that favors the longest name. - :param package: Prefix string "name.name.name" + :param package: Prefix string "path.path.path" :param name: The variable we're looking for - :return: Name resolution as a Rereferent, often a value, but maybe a package or an + :return: Name resolution as a ``Rereferent``, often a value, but maybe a package or an annotation. + :raises KeyError: if the name cannot be found in this ``NameContainer`` """ self.logger.debug( "resolve_name(%r.%r) in %s, parent=%s", package, name, - self.keys, + list(self.keys()), self.parent, ) # Longest Name @@ -893,27 +1069,30 @@ def resolve_name(self, package: Optional[str], name: str) -> Referent: else: target = [""] # Pool of matches - matches: List[Tuple[List[str], Union["NameContainer", Result]]] = [] + matches: List[Tuple[List[str], Referent]] = [] # Target has an extra item to make the len non-zero. while not matches and target: target = target[:-1] - for p in self.parent_iter(): + for nc in self.parent_iter(): try: package_ident: List[str] = target + [name] - match: Union["NameContainer", Result] = p.find_name(package_ident) - matches.append((package_ident, match)) + # Find the Referent for this name. + ref_to = nc.find_name(package_ident) + matches.append((package_ident, ref_to)) except NameContainer.NotFound: # No matches; move to the parent and try again. pass self.logger.debug( "resolve_name: target=%s+[%r], matches=%s", target, name, matches ) + # NOTE: There are two separate kinds of failures: no name at all, and no value for the name. + # This is the no name at all. The collaborator may need the value or the annotation. if not matches: raise KeyError(name) + # Find the longest name match and return the Referent. # This feels hackish -- it should be the first referent value. - # Find the longest name match.p - path, value = max(matches, key=lambda path_value: len(path_value[0])) - return cast(Referent, value) + path, best_match = max(matches, key=lambda path_value: len(path_value[0])) + return best_match def clone(self) -> "NameContainer": new = NameContainer(parent=self.parent) @@ -921,6 +1100,20 @@ def clone(self) -> "NameContainer": new[k] = v.clone() return new + def get( # type: ignore[override] + self, name: str, default: Optional[Referent] = None + ) -> Union[ + Annotation, celpy.celtypes.Value, CELEvalError, CELFunction, "NameContainer" + ]: + """ + Used by transpiled code to get values from a NameContainer of Referents. + + .. important:: This does not get a Referent, it gets a value. + + .. todo:: This is a poorly-chosen name; a number of related types **all** need to have a get_value() method. + """ + return self.resolve_name(None, name).value + def __repr__(self) -> str: return f"{self.__class__.__name__}({dict(self)}, parent={self.parent})" @@ -928,6 +1121,7 @@ def __repr__(self) -> str: class Activation: """ Namespace with variable bindings and type name ("annotation") bindings. + Additionally, the pool of functions and types are here, also. .. rubric:: Life and Content @@ -937,22 +1131,22 @@ class Activation: A nested Activation is created each time we evaluate a macro. - An Activation contains a ``NameContainer`` instance to resolve identifers. + An Activation contains a ``NameContainer`` instance to resolve identifiers. (This may be a needless distinction and the two classes could, perhaps, be combined.) - .. todo:: The environment's annotations are type names used for protobuf. + These names include variables as well as type names used for protobuf and the internal CEL ``type()`` function. .. rubric:: Chaining/Nesting Activations can form a chain so locals are checked first. Activations can nest via macro evaluation, creating transient local variables. + Consider this CEL macro expression: :: ``"[2, 4, 6].map(n, n / 2)"`` - means nested activations with ``n`` bound to 2, 4, and 6 respectively. - The resulting objects then form a resulting list. + This works via a nested activation with ``n`` bound to 2, 4, and 6 respectively. This is used by an :py:class:`Evaluator` as follows:: @@ -989,84 +1183,111 @@ class Activation: "namespace resolution should try to find the longest prefix for the evaluator." Most names start with ``IDENT``, but a primary can start with ``.``. + A leading ``.`` changes the search order from most local first to root first. """ def __init__( self, + *, # Keyword only, too many things here. annotations: Optional[Mapping[str, Annotation]] = None, - package: Optional[str] = None, vars: Optional[Context] = None, - parent: Optional["Activation"] = None, + functions: Optional[Union[Mapping[str, CELFunction], list[CELFunction]]] = None, + package: Optional[str] = None, + based_on: Optional["Activation"] = None, ) -> None: """ Create an Activation. - The annotations are loaded first. The variables are loaded second, and placed - in front of the annotations in the chain of name resolutions. Values come before - annotations. + The annotations are loaded first. The variables and their values are loaded second, and placed + in front of the annotations in the chain of name resolutions. - :param annotations: Variables and type annotations. + The Evaluator and the Transpiler use this to resolve identifiers into types, values, or functions. + + :keyword annotations: Variables and type annotations. Annotations are loaded first to serve as defaults to create a parent NameContainer. - :param package: The package name to assume as a prefix for name resolution. - :param vars: Variables and their values, loaded to update the NameContainer. - :param parent: A parent activation in the case of macro evaluations. + :keyword vars: Variables and their values, loaded to update the NameContainer. + :keyword functions: functions and their implementation, loaded to update the NameContainer. + :keyword package: The package name to assume as a prefix for name resolution. + :keyword based_on: A foundational activation on which this is based. """ logger.debug( - "Activation(annotations=%r, package=%r, vars=%r, parent=%s)", + "Activation(annotations=%r, vars=%r, functions=%r, package=%r, based_on=%s)", annotations, - package, vars, - parent, + functions, + package, + based_on, ) - # Seed the annotation identifiers for this activation. + # Seed the annotations for identifiers in this activation. self.identifiers: NameContainer = NameContainer( - parent=parent.identifiers if parent else None + parent=based_on.identifiers if based_on else None ) if annotations is not None: self.identifiers.load_annotations(annotations) + if vars is not None: + # Set values from a dictionary of names and values. + self.identifiers.load_values(vars) + + # Update this NameContainer functions (if any.) + self.functions: collections.ChainMap[str, CELFunction] + if isinstance(functions, Sequence): + local_functions: dict[str, CELFunction] = { + f.__name__: f for f in functions or [] + } + self.functions = collections.ChainMap(local_functions, base_functions) + # self.identifiers.load_values(local_functions) + elif isinstance(functions, Mapping): + self.functions = collections.ChainMap( + cast(dict[str, CELFunction], functions), base_functions + ) + elif functions is None: + self.functions = collections.ChainMap(base_functions) + else: + raise ValueError("functions not a mapping or sequence") # pragma: no cover # The name of the run-time package -- an assumed prefix for name resolution self.package = package - # Create a child NameContainer with variables (if any.) - if vars is None: - pass - elif isinstance(vars, Activation): # pragma: no cover - # Deprecated legacy feature. - raise NotImplementedError("Use Activation.clone()") - - else: - # Set values from a dictionary of names and values. - self.identifiers.load_values(vars) - def clone(self) -> "Activation": """ Create a clone of this activation with a deep copy of the identifiers. """ + logger.debug("Cloning an Activation...") clone = Activation() - clone.package = self.package clone.identifiers = self.identifiers.clone() + clone.functions = self.functions.copy() + clone.package = self.package + logger.debug("clone: %r", self) return clone def nested_activation( self, - annotations: Optional[Mapping[str, Annotation]] = None, + annotations: Optional[Mapping[str, Annotation]] = None, # Remove this. vars: Optional[Context] = None, ) -> "Activation": """ - Create a nested sub-Activation that chains to the current activation. - The sub-activations don't have the same implied package context, - - :param annotations: Variable type annotations - :param vars: Variables with literals to be converted to the desired types. - :return: An ``Activation`` that chains to this Activation. - """ - new = Activation( - annotations=annotations, vars=vars, parent=self, package=self.package + Create an Activation based on the current activation. + This new Activation will be seeded from the current activation's + ``NameContainer``. + + :param annotations: Optional type definitions for the new local variables. + :param vars: Local variables to be added when creating this activation. + :return: A subsidiary ``Activation`` that chains to this Activation. + """ + logger.debug("Creating nested Activation...") + nested = Activation( + annotations=annotations, # Replace with self.annotations. + vars=vars, + functions=self.functions, + package=self.package, + based_on=self, ) - return new + logger.debug("nested: %r", self) + return nested - def resolve_variable(self, name: str) -> Union[Result, NameContainer]: + def resolve_variable( + self, name: str + ) -> Union[celpy.celtypes.Value, CELFunction, NameContainer]: """Find the object referred to by the name. An Activation usually has a chain of NameContainers to be searched. @@ -1074,9 +1295,55 @@ def resolve_variable(self, name: str) -> Union[Result, NameContainer]: A variable can refer to an annotation and/or a value and/or a nested container. Most of the time, we want the `value` attribute of the Referent. This can be a Result (a Union[Value, CelType]) + + There's a subtle difference between a variable without an annotation, + and a variable with an annotation, but without a value. """ - container_or_value = self.identifiers.resolve_name(self.package, str(name)) - return cast(Union[Result, NameContainer], container_or_value) + # Will be a Referent. Get Value or Type -- interpreter works with either. + logger.debug("resolve_variable(%r)", name) + try: + referent = self.identifiers.resolve_name(self.package, name) + return cast(Union[Result, NameContainer], referent.value) + except KeyError: + return self.functions[name] + + def resolve_function( + self, name: str + ) -> Union[CELFunction, celpy.celtypes.TypeType]: + """A short-cut to find functions without looking at Variables first.""" + logger.debug("resolve_function(%r)", name) + return self.functions[name] + + def __getattr__( + self, name: str + ) -> Union[celpy.celtypes.Value, CELFunction, NameContainer]: + """Handle ``activation.name`` in transpiled code (or ``activation.get('name')``). + + If the name is not in the Activation with a value, a ``NameError`` exception must be raised. + + Note that :py:meth:`Activation.resolve_variable` depends on :py:meth:`NameContainer.find_name`. + The :py:meth:`NameContainer.find_name` method **also** find the value. + + This is -- perhaps -- less than optimal because it can mask the no value set case. + """ + # Will be a Referent. Get Value if it was set or raise error if no value set. + try: + referent = self.identifiers.resolve_name(self.package, name) + logger.debug("get/__getattr__(%r) ==> %r", name, referent) + if referent._value_set: + return cast(Union[Result, NameContainer], referent.value) + else: + if referent.container: + return referent.container + elif referent.annotation: + return cast(Union[Result, NameContainer], referent.annotation) + else: + raise RuntimeError(f"Corrupt {self!r}") # pragma: no cover + except KeyError: + logger.debug("get/__getattr__(%r) fallback to functions", name) + return self.functions[name] + + get = __getattr__ def __repr__(self) -> str: return ( @@ -1084,33 +1351,11 @@ def __repr__(self) -> str: f"(annotations={self.identifiers.parent!r}, " f"package={self.package!r}, " f"vars={self.identifiers!r}, " + f"functions={self.functions!r}, " f"parent={self.identifiers.parent})" ) -class FindIdent(lark.visitors.Visitor_Recursive): - """Locate the ident token at the bottom of an AST. - - This is needed to find the bind variable for macros. - - It works by doing a "visit" on the entire tree, but saving - the details of the ``ident`` nodes only. - """ - - def __init__(self) -> None: - self.ident_token: Optional[str] = None - - def ident(self, tree: lark.Tree) -> None: - ident_token = cast(lark.Token, tree.children[0]) - self.ident_token = ident_token.value - - @classmethod - def in_tree(cls: Type["FindIdent"], tree: lark.Tree) -> Optional[str]: - fi = FindIdent() - fi.visit(tree) - return fi.ident_token - - def trace( method: Callable[["Evaluator", lark.Tree], Any], ) -> Callable[["Evaluator", lark.Tree], Any]: @@ -1124,9 +1369,9 @@ def trace( @wraps(method) def concrete_method(self: "Evaluator", tree: lark.Tree) -> Any: self.logger.debug("%s%r", self.level * "| ", tree) - result = method(self, tree) - self.logger.debug("%s%s -> %r", self.level * "| ", tree.data, result) - return result + result_value = method(self, tree) + self.logger.debug("%s%s -> %r", self.level * "| ", tree.data, result_value) + return result_value if os.environ.get("CEL_TRACE"): return concrete_method @@ -1145,45 +1390,48 @@ class Evaluator(lark.visitors.Interpreter[Result]): An AST node must call ``self.visit_children(tree)`` explicitly to build the values for all the children of this node. - Exceptions. + .. rubric:: Exceptions To handle ``2 / 0 || true``, the ``||``, ``&&``, and ``?:`` operators do not trivially evaluate and raise exceptions. They bottle up the exceptions and treat them as a kind of undecided value. - Identifiers. + .. rubric:: Identifiers Identifiers have three meanings: - An object. This is either a variable provided in the activation or a function provided when building an execution. Objects also have type annotations. - - A type annotation without an object, This is used to build protobuf messages. + - A type annotation without an object. This is used to build protobuf messages. - A macro name. The ``member_dot_arg`` construct may have a macro. Plus the ``ident_arg`` construct may also have a ``dyn()`` or ``has()`` macro. See below for more. - Other than macros, a name maps to an ``Referent`` instance. This will have an - annotation and -- perhaps -- an associated object. + Other than macros, a name maps to an ``Referent`` instance. + This will have an annotation and -- perhaps -- an associated object. Names have nested paths. ``a.b.c`` is a mapping, ``a``, that contains a mapping, ``b``, that contains ``c``. - **MACROS ARE SPECIAL**. + .. important MACROS ARE SPECIAL + + They aren't simple functions. + + The macros do not **all** simply visit their children to perform evaluation. - The macros do not **all** simply visit their children to perform evaluation. There are three cases: - ``dyn()`` does effectively nothing. - It visits it's children, but also provides progressive type resolution + It visits its children, but also provides progressive type resolution through annotation of the AST. - ``has()`` attempts to visit the child and does a boolean transformation on the result. This is a macro because it doesn't raise an exception for a missing member item reference, but instead maps an exception to False. - It doesn't return the value found, for a member item reference; instead, it maps + It doesn't return the value found for a member item reference; instead, it maps this to True. - The various ``member.macro()`` constructs do **NOT** visit children. @@ -1203,7 +1451,7 @@ def __init__( self, ast: lark.Tree, activation: Activation, - functions: Union[Sequence[CELFunction], Mapping[str, CELFunction], None] = None, + # functions: Union[Sequence[CELFunction], Mapping[str, CELFunction], None] = None, # Refactor into Activation ) -> None: """ Create an evaluator for an AST with specific variables and functions. @@ -1211,39 +1459,32 @@ def __init__( :param ast: The AST to evaluate. :param activation: The variable bindings to use. :param functions: The functions to use. If nothing is supplied, the default - global `base_functions` are used. Otherwise a ChainMap is created so + global `base_functions` are used. Otherwise, a ``ChainMap`` is created so these local functions override the base functions. """ self.ast = ast self.base_activation = activation self.activation = self.base_activation - self.functions: Mapping[str, CELFunction] - if isinstance(functions, Sequence): - local_functions = {f.__name__: f for f in functions or []} - self.functions = collections.ChainMap(local_functions, base_functions) # type: ignore [arg-type] - elif isinstance(functions, Mapping): - self.functions = collections.ChainMap(functions, base_functions) # type: ignore [arg-type] - else: - self.functions = base_functions self.level = 0 - self.logger.debug("activation: %r", self.activation) - self.logger.debug("functions: %r", self.functions) + self.logger.debug("Evaluator activation: %r", self.activation) + # self.logger.debug("functions: %r", self.functions) # Refactor ``self.functions`` into an Activation def sub_evaluator(self, ast: lark.Tree) -> "Evaluator": """ Build an evaluator for a sub-expression in a macro. + :param ast: The AST for the expression in the macro. :return: A new `Evaluator` instance. """ - return Evaluator(ast, activation=self.activation, functions=self.functions) + return Evaluator(ast, activation=self.activation) def set_activation(self, values: Context) -> "Evaluator": """ - Chain a new activation using the given Context. + Create a new activation using the given Context. This is used for two things: - 1. Bind external variables like command-line arguments or environment variables. + 1. Bind external variables. Examples are command-line arguments and environment variables. 2. Build local variable(s) for macro evaluation. """ @@ -1259,13 +1500,14 @@ def ident_value(self, name: str, root_scope: bool = False) -> Result_Function: We may be limited to root scope, which prevents searching through alternative protobuf package definitions. + In principle, this changes the order of the search. """ - try: - return cast(Result, self.activation.resolve_variable(name)) - except KeyError: - return self.functions[name] + # try: + return cast(Result, self.activation.resolve_variable(name)) + # except KeyError: + # return self.functions[name] # Refactor ``self.functions`` into an Activation - def evaluate(self) -> celpy.celtypes.Value: + def evaluate(self, context: Optional[Context] = None) -> celpy.celtypes.Value: """ Evaluate this AST and return the value or raise an exception. @@ -1273,9 +1515,11 @@ def evaluate(self) -> celpy.celtypes.Value: - External clients want the value or the exception. - - Internally, we sometimes want to silence CELEvalError exceptions so that + - Internally, we sometimes want to silence ``CELEvalError`` exceptions so that we can apply short-circuit logic and choose a non-exceptional result. """ + if context: + self.set_activation(context) value = self.visit(self.ast) if isinstance(value, CELEvalError): raise value @@ -1284,9 +1528,9 @@ def evaluate(self) -> celpy.celtypes.Value: def visit_children(self, tree: lark.Tree) -> List[Result]: """Extend the superclass to track nesting and current evaluation context.""" self.level += 1 - result = super().visit_children(tree) + result_value = super().visit_children(tree) self.level -= 1 - return result + return result_value def function_eval( self, name_token: lark.Token, exprlist: Optional[Iterable[Result]] = None @@ -1295,13 +1539,14 @@ def function_eval( Function evaluation. - Object creation and type conversions. - - Other built-in functions like size() + - Other functions like ``size()`` or ``type()`` - Extension functions """ function: CELFunction try: # TODO: Transitive Lookup of function in all parent activation contexts. - function = self.functions[name_token.value] + # function = self.functions[name_token.value] # Refactor ``self.functions`` into an Activation + function = self.activation.resolve_function(name_token.value) except KeyError as ex: err = ( f"undeclared reference to '{name_token}' " @@ -1338,18 +1583,23 @@ def method_eval( exprlist: Optional[Iterable[Result]] = None, ) -> Result: """ - Method evaluation. While are (nominally) attached to an object, the only thing - actually special is that the object is the first parameter to a function. + Method evaluation. While these are (nominally) attached to an object, + that would make overrides complicated. + Instead, these are functions (which can be overridden). + The object must the first parameter to a function. """ function: CELFunction try: # TODO: Transitive Lookup of function in all parent activation contexts. - function = self.functions[method_ident.value] + # function = self.functions[method_ident.value] # Refactor ``self.functions`` into an Activation + function = self.activation.resolve_function(method_ident.value) except KeyError as ex: self.logger.debug( "method_eval(%r, %r, %s) --> %r", object, method_ident, exprlist, ex ) - self.logger.debug("functions: %s", self.functions) + self.logger.debug( + "functions: %s", self.activation.functions + ) # Refactor ``self.functions`` into an Activation err = ( f"undeclared reference to {method_ident.value!r} " f"(in activation '{self.activation}')" @@ -1399,14 +1649,14 @@ def macro_has_eval(self, exprlist: lark.Tree) -> celpy.celtypes.BoolType: - If f is a repeated field or map field, has(e.f) indicates whether the field is non-empty. - - If f is a singular or oneof field, has(e.f) indicates whether the field is set. + - If f is a singular or "oneof" field, has(e.f) indicates whether the field is set. 4. If e evaluates to a protocol buffers version 3 message and f is a defined field: - If f is a repeated field or map field, has(e.f) indicates whether the field is non-empty. - - If f is a oneof or singular message field, has(e.f) indicates whether the field + - If f is a "oneof" field or singular message field, has(e.f) indicates whether the field is set. - If f is some other singular field, has(e.f) indicates whether the field's value @@ -1426,7 +1676,7 @@ def expr(self, tree: lark.Tree) -> Result: The default implementation short-circuits and can ignore a CELEvalError in the two alternative sub-expressions. - The conditional sub-expression CELEvalError is propogated out as the result. + The conditional sub-expression CELEvalError is propagated out as the result. See https://github.com/google/cel-spec/blob/master/doc/langdef.md#logical-operators @@ -1441,7 +1691,8 @@ def expr(self, tree: lark.Tree) -> Result: return values[0] elif len(tree.children) == 3: # full conditionalor "?" conditionalor ":" expr. - func = self.functions["_?_:_"] + # func = self.functions["_?_:_"] # Refactor ``self.functions`` into an Activation + func = self.activation.resolve_function("_?_:_") cond_value = self.visit(cast(lark.Tree, tree.children[0])) left = right = cast(Result, celpy.celtypes.BoolType(False)) try: @@ -1479,7 +1730,8 @@ def conditionalor(self, tree: lark.Tree) -> Result: values = self.visit_children(tree) return values[0] elif len(tree.children) == 2: - func = self.functions["_||_"] + # func = self.functions["_||_"] # Refactor ``self.functions`` into an Activation + func = self.activation.resolve_function("_||_") left, right = cast(Tuple[Result, Result], self.visit_children(tree)) try: return func(left, right) @@ -1512,7 +1764,8 @@ def conditionaland(self, tree: lark.Tree) -> Result: values = self.visit_children(tree) return values[0] elif len(tree.children) == 2: - func = self.functions["_&&_"] + # func = self.functions["_&&_"] # Refactor ``self.functions`` into an Activation + func = self.activation.resolve_function("_&&_") left, right = cast(Tuple[Result, Result], self.visit_children(tree)) try: return func(left, right) @@ -1552,7 +1805,7 @@ def relation(self, tree: lark.Tree) -> Result: values = self.visit_children(tree) func = functions[op_name_map[tree.data]] - result = func(*values) + result_value = func(*values) The AST doesn't provide a flat list of values, however. """ @@ -1563,6 +1816,7 @@ def relation(self, tree: lark.Tree) -> Result: elif len(tree.children) == 2: left_op, right_tree = cast(Tuple[lark.Tree, lark.Tree], tree.children) + # Map a node data in parse tree to an operation function. op_name = { "relation_lt": "_<_", "relation_le": "_<=_", @@ -1572,7 +1826,8 @@ def relation(self, tree: lark.Tree) -> Result: "relation_ne": "_!=_", "relation_in": "_in_", }[left_op.data] - func = self.functions[op_name] + # func = self.functions[op_name] # Refactor ``self.functions`` into an Activation + func = self.activation.resolve_function(op_name) # NOTE: values have the structure [[left], right] (left, *_), right = cast( Tuple[List[Result], Result], self.visit_children(tree) @@ -1611,7 +1866,7 @@ def addition(self, tree: lark.Tree) -> Result: values = self.visit_children(tree) func = functions[op_name_map[tree.data]] - result = func(*values) + result_value = func(*values) The AST doesn't provide a flat list of values, however. """ @@ -1622,11 +1877,13 @@ def addition(self, tree: lark.Tree) -> Result: elif len(tree.children) == 2: left_op, right_tree = cast(Tuple[lark.Tree, lark.Tree], tree.children) + # Map a node data in parse tree to an operation function. op_name = { "addition_add": "_+_", "addition_sub": "_-_", }[left_op.data] - func = self.functions[op_name] + # func = self.functions[op_name] # Refactor ``self.functions`` into an Activation + func = self.activation.resolve_function(op_name) # NOTE: values have the structure [[left], right] (left, *_), right = cast( Tuple[List[Result], Result], self.visit_children(tree) @@ -1673,7 +1930,7 @@ def multiplication(self, tree: lark.Tree) -> Result: values = self.visit_children(tree) func = functions[op_name_map[tree.data]] - result = func(*values) + result_value = func(*values) The AST doesn't provide a flat list of values, however. """ @@ -1684,12 +1941,14 @@ def multiplication(self, tree: lark.Tree) -> Result: elif len(tree.children) == 2: left_op, right_tree = cast(Tuple[lark.Tree, lark.Tree], tree.children) + # Map a node data in parse tree to an operation function. op_name = { "multiplication_div": "_/_", "multiplication_mul": "_*_", "multiplication_mod": "_%_", }[left_op.data] - func = self.functions[op_name] + # func = self.functions[op_name] # Refactor ``self.functions`` into an Activation + func = self.activation.resolve_function(op_name) # NOTE: values have the structure [[left], right] (left, *_), right = cast( Tuple[List[Result], Result], self.visit_children(tree) @@ -1742,23 +2001,25 @@ def unary(self, tree: lark.Tree) -> Result: values = self.visit_children(tree) func = functions[op_name_map[tree.data]] - result = func(*values) + result_value = func(*values) But, values has the structure ``[[], right]`` """ if len(tree.children) == 1: - # member with no preceeding unary_not or unary_neg + # member with no preceding unary_not or unary_neg # TODO: If there are two possible values (namespace v. mapping) chose the namespace. values = self.visit_children(tree) return values[0] elif len(tree.children) == 2: op_tree, right_tree = cast(Tuple[lark.Tree, lark.Tree], tree.children) + # Map a node data in parse tree to an operation function. op_name = { "unary_not": "!_", "unary_neg": "-_", }[op_tree.data] - func = self.functions[op_name] + # func = self.functions[op_name] # Refactor ``self.functions`` into an Activation + func = self.activation.resolve_function(op_name) # NOTE: values has the structure [[], right] left, right = cast(Tuple[List[Result], Result], self.visit_children(tree)) self.logger.debug("unary %s %r", op_name, right) @@ -1813,19 +2074,19 @@ def build_macro_eval( """ args = cast(lark.Tree, child.children[2]) var_tree, expr_tree = cast(Tuple[lark.Tree, lark.Tree], args.children) - identifier = FindIdent.in_tree(var_tree) - if identifier is None: # pragma: no cover - # This seems almost impossible. - raise CELSyntaxError( + idents = list(var_tree.find_data("ident")) + if len(idents) != 1: + # Essentially impossible. + raise CELSyntaxError( # pragma: no cover f"{child.data} {child.children}: bad macro node", line=child.meta.line, column=child.meta.column, ) - # nested_eval = Evaluator(ast=expr_tree, activation=self.activation) + identifier = cast(lark.Token, idents[0].children[0]).value nested_eval = self.sub_evaluator(ast=expr_tree) def sub_expr(v: celpy.celtypes.Value) -> Any: - return nested_eval.set_activation({identifier: v}).evaluate() + return nested_eval.evaluate({identifier: v}) return sub_expr @@ -1845,20 +2106,29 @@ def build_ss_macro_eval( """ args = cast(lark.Tree, child.children[2]) var_tree, expr_tree = cast(Tuple[lark.Tree, lark.Tree], args.children) - identifier = FindIdent.in_tree(var_tree) - if identifier is None: # pragma: no cover - # This seems almost impossible. - raise CELSyntaxError( + idents = list(var_tree.find_data("ident")) + if len(idents) != 1: + # Essentially impossible. + raise CELSyntaxError( # pragma: no cover f"{child.data} {child.children}: bad macro node", line=child.meta.line, column=child.meta.column, ) + identifier = cast(lark.Token, idents[0].children[0]).value + # identifier = FindIdent.in_tree(var_tree) + # if identifier is None: # pragma: no cover + # # This seems almost impossible. + # raise CELSyntaxError( + # f"{child.data} {child.children}: bad macro node", + # line=child.meta.line, + # column=child.meta.column, + # ) # nested_eval = Evaluator(ast=expr_tree, activation=self.activation) nested_eval = self.sub_evaluator(ast=expr_tree) def sub_expr(v: celpy.celtypes.Value) -> Any: try: - return nested_eval.set_activation({identifier: v}).evaluate() + return nested_eval.evaluate({identifier: v}) except CELEvalError as ex: return ex @@ -1868,7 +2138,7 @@ def build_reduce_macro_eval( self, child: lark.Tree ) -> Tuple[Callable[[Result, Result], Result], lark.Tree]: """ - Builds macro function and intiial expression for reduce(). + Builds macro function and initial expression for reduce(). For example @@ -1889,22 +2159,31 @@ def build_reduce_macro_eval( reduce_var_tree, iter_var_tree, init_expr_tree, expr_tree = cast( Tuple[lark.Tree, lark.Tree, lark.Tree, lark.Tree], args.children ) - reduce_ident = FindIdent.in_tree(reduce_var_tree) - iter_ident = FindIdent.in_tree(iter_var_tree) - if reduce_ident is None or iter_ident is None: # pragma: no cover + reduce_idents = list(reduce_var_tree.find_data("ident")) + iter_idents = list(iter_var_tree.find_data("ident")) + if len(reduce_idents) != 1 or len(iter_idents) != 1: # pragma: no cover # This seems almost impossible. raise CELSyntaxError( f"{child.data} {child.children}: bad macro node", line=child.meta.line, column=child.meta.column, ) + reduce_ident = cast(lark.Token, reduce_idents[0].children[0]).value + iter_ident = cast(lark.Token, iter_idents[0].children[0]).value + # reduce_ident = FindIdent.in_tree(reduce_var_tree) + # iter_ident = FindIdent.in_tree(iter_var_tree) + # if reduce_ident is None or iter_ident is None: # pragma: no cover + # # This seems almost impossible. + # raise CELSyntaxError( + # f"{child.data} {child.children}: bad macro node", + # line=child.meta.line, + # column=child.meta.column, + # ) # nested_eval = Evaluator(ast=expr_tree, activation=self.activation) nested_eval = self.sub_evaluator(ast=expr_tree) def sub_expr(r: Result, i: Result) -> Result: - return nested_eval.set_activation( - {reduce_ident: r, iter_ident: i} - ).evaluate() + return nested_eval.evaluate({reduce_ident: r, iter_ident: i}) return sub_expr, init_expr_tree @@ -1967,46 +2246,46 @@ def member_dot(self, tree: lark.Tree) -> Result: - In all other cases, e.f evaluates to an error. - TODO: implement member "." IDENT for messages. + TODO: implement member "." IDENT for protobuf message types. """ member_tree, property_name_token = cast( Tuple[lark.Tree, lark.Token], tree.children ) member = self.visit(member_tree) property_name = property_name_token.value - result: Result + result_value: Result if isinstance(member, CELEvalError): - result = cast(Result, member) + result_value = cast(Result, member) elif isinstance(member, NameContainer): # Navigation through names provided as external run-time bindings. # The dict is the value of a Referent that was part of a namespace path. if property_name in member: - result = cast(Result, member[property_name].value) + result_value = cast(Result, member[property_name].value) else: err = f"No {property_name!r} in bindings {sorted(member.keys())}" - result = CELEvalError(err, KeyError, None, tree=tree) - # TODO: Not sure this is needed... + result_value = CELEvalError(err, KeyError, None, tree=tree) elif isinstance(member, celpy.celtypes.MessageType): + # NOTE: Message's don't have a "default None" behavior: they raise an exception. self.logger.debug("member_dot(%r, %r)", member, property_name) - result = member.get(property_name) + result_value = member.get(property_name) # TODO: Future Expansion, handle Protobuf message package... # elif isinstance(member, celpy.celtypes.PackageType): # if property_name in member: - # result = member[property_name] + # result_value = member[property_name] # else: # err = f"no such message {property_name!r} in package {member}" - # result = CELEvalError(err, KeyError, None, tree=tree) + # result_value = CELEvalError(err, KeyError, None, tree=tree) elif isinstance(member, celpy.celtypes.MapType): - # Syntactic sugar: a.b is a["b"] when a is a mapping. + # Syntactic sugar: a.b is a["b"] when ``a`` is a mapping. try: - result = member[property_name] + result_value = member[property_name] except KeyError: err = f"no such member in mapping: {property_name!r}" - result = CELEvalError(err, KeyError, None, tree=tree) + result_value = CELEvalError(err, KeyError, None, tree=tree) else: err = f"{member!r} with type: '{type(member)}' does not support field selection" - result = CELEvalError(err, TypeError, None, tree=tree) - return result + result_value = CELEvalError(err, TypeError, None, tree=tree) + return result_value @trace def member_dot_arg(self, tree: lark.Tree) -> Result: @@ -2029,7 +2308,7 @@ def member_dot_arg(self, tree: lark.Tree) -> Result: - member "." IDENT "(" ")" -- used for a several timestamp operations. """ sub_expr: CELFunction - result: Result + result_value: Result reduction: Result CELBoolFunction = Callable[ [celpy.celtypes.BoolType, Result], celpy.celtypes.BoolType @@ -2045,9 +2324,13 @@ def member_dot_arg(self, tree: lark.Tree) -> Result: "all", "exists", "exists_one", + # Extensions to CEL... "reduce", "min", }: + # TODO: These can be refactored to share the macro_xyz() functions + # used by Transpiled code. + member_list = cast(celpy.celtypes.ListType, self.visit(member_tree)) if isinstance(member_list, CELEvalError): @@ -2058,13 +2341,13 @@ def member_dot_arg(self, tree: lark.Tree) -> Result: mapping = cast( Iterable[celpy.celtypes.Value], map(sub_expr, member_list) ) - result = celpy.celtypes.ListType(mapping) - return result + result_value = celpy.celtypes.ListType(mapping) + return result_value elif method_name_token.value == "filter": sub_expr = self.build_macro_eval(tree) - result = celpy.celtypes.ListType(filter(sub_expr, member_list)) - return result + result_value = celpy.celtypes.ListType(filter(sub_expr, member_list)) + return result_value elif method_name_token.value == "all": sub_expr = self.build_ss_macro_eval(tree) @@ -2098,6 +2381,7 @@ def member_dot_arg(self, tree: lark.Tree) -> Result: count = sum(1 for value in member_list if bool(sub_expr(value))) return celpy.celtypes.BoolType(count == 1) + # Not formally part of CEL... elif method_name_token.value == "reduce": # Apply a function to reduce the list to a single value. # The `tree` is a `member_dot_arg` construct with (member, method_name, args) @@ -2107,6 +2391,7 @@ def member_dot_arg(self, tree: lark.Tree) -> Result: reduction = reduce(reduce_expr, member_list, initial_value) return reduction + # Not formally part of CEL... elif method_name_token.value == "min": # Special case of "reduce()" # with .min() -> .reduce(r, i, int_max, r < i ? r : i) @@ -2128,15 +2413,15 @@ def member_dot_arg(self, tree: lark.Tree) -> Result: member, ident = cast( Tuple[Result, lark.Token], self.visit_children(tree) ) - result = self.method_eval(member, ident) + result_value = self.method_eval(member, ident) else: # assert len(tree.children) == 3 member, ident, expr_iter = cast( Tuple[Result, lark.Token, Iterable[Result]], self.visit_children(tree), ) - result = self.method_eval(member, ident, expr_iter) - return result + result_value = self.method_eval(member, ident, expr_iter) + return result_value @trace def member_index(self, tree: lark.Tree) -> Result: @@ -2152,7 +2437,8 @@ def member_index(self, tree: lark.Tree) -> Result: Locating an item in a Mapping or List """ - func = self.functions["_[_]"] + # func = self.functions["_[_]"] # Refactor ``self.functions`` into an Activation + func = self.activation.resolve_function("_[_]") values = self.visit_children(tree) member, index = values try: @@ -2246,15 +2532,16 @@ def primary(self, tree: lark.Tree) -> Result: list_lit : "[" [exprlist] "]" map_lit : "{" [mapinits] "}" - TODO: Refactor into separate methods to skip this complex elif chain. - top-level :py:meth:`primary` is similar to :py:meth:`method`. - Each of the individual rules then works with a tree instead of a child of the - primary tree. + .. TODO:: Refactor into separate methods to skip these complex elif chain. + + Top-level :py:meth:`primary` is similar to :py:meth:`method`. + Each of the individual rules then works with a tree instead of a child of the + primary tree. - This includes function-like macros: has() and dyn(). + This includes function-like macros: ``has()`` and ``dyn()``. These are special cases and cannot be overridden. """ - result: Result + result_value: Result name_token: lark.Token if len(tree.children) != 1: raise CELSyntaxError( @@ -2278,18 +2565,18 @@ def primary(self, tree: lark.Tree) -> Result: if len(child.children) == 0: # Empty list # TODO: Refactor into type_eval() - result = celpy.celtypes.ListType() + result_value = celpy.celtypes.ListType() else: # exprlist to be packaged as List. values = self.visit_children(child) - result = values[0] - return result + result_value = values[0] + return result_value elif child.data == "map_lit": if len(child.children) == 0: # Empty mapping # TODO: Refactor into type_eval() - result = celpy.celtypes.MapType() + result_value = celpy.celtypes.MapType() else: # mapinits (a sequence of key-value tuples) to be packaged as a dict. # OR. An CELEvalError in case of ValueError caused by duplicate keys. @@ -2297,30 +2584,36 @@ def primary(self, tree: lark.Tree) -> Result: # TODO: Refactor into type_eval() try: values = self.visit_children(child) - result = values[0] + result_value = values[0] except ValueError as ex: - result = CELEvalError(ex.args[0], ex.__class__, ex.args, tree=tree) + result_value = CELEvalError( + ex.args[0], ex.__class__, ex.args, tree=tree + ) except TypeError as ex: - result = CELEvalError(ex.args[0], ex.__class__, ex.args, tree=tree) - return result + result_value = CELEvalError( + ex.args[0], ex.__class__, ex.args, tree=tree + ) + return result_value elif child.data in ("dot_ident", "dot_ident_arg"): # "." IDENT ["(" [exprlist] ")"] # Leading "." means the name is resolved in the root scope **only**. - # No searching through alterantive packages. + # No searching through alternative packages. # len(child) == 1 -- "." IDENT # len(child) == 2 -- "." IDENT "(" exprlist ")" -- TODO: Implement dot_ident_arg. values = self.visit_children(child) name_token = cast(lark.Token, values[0]) # Should not be a Function, should only be a Result - # TODO: implement dot_ident_arg uses function_eval(). + # TODO: implement dot_ident_arg using ``function_eval()``, which should match this code. try: - result = cast( + result_value = cast( Result, self.ident_value(name_token.value, root_scope=True) ) except KeyError as ex: - result = CELEvalError(ex.args[0], ex.__class__, ex.args, tree=tree) - return result + result_value = CELEvalError( + ex.args[0], ex.__class__, ex.args, tree=tree + ) + return result_value elif child.data == "ident_arg": # IDENT ["(" [exprlist] ")"] @@ -2362,14 +2655,14 @@ def primary(self, tree: lark.Tree) -> Result: # Should not be a Function. # Generally Result object (i.e., a variable) # Could be an Annotation object (i.e., a type) for protobuf messages - result = cast(Result, self.ident_value(name_token.value)) + result_value = cast(Result, self.ident_value(name_token.value)) except KeyError as ex: err = ( f"undeclared reference to '{name_token}' " f"(in activation '{self.activation}')" ) - result = CELEvalError(err, ex.__class__, ex.args, tree=tree) - return result + result_value = CELEvalError(err, ex.__class__, ex.args, tree=tree) + return result_value else: raise CELSyntaxError( @@ -2393,11 +2686,11 @@ def literal(self, tree: lark.Tree) -> Result: ) value_token = cast(lark.Token, tree.children[0]) try: - result: Result + result_value: Result if value_token.type == "FLOAT_LIT": - result = celpy.celtypes.DoubleType(value_token.value) + result_value = celpy.celtypes.DoubleType(value_token.value) elif value_token.type == "INT_LIT": - result = celpy.celtypes.IntType(value_token.value) + result_value = celpy.celtypes.IntType(value_token.value) elif value_token.type == "UINT_LIT": if not value_token.value[-1].lower() == "u": raise CELSyntaxError( @@ -2405,15 +2698,17 @@ def literal(self, tree: lark.Tree) -> Result: line=tree.meta.line, column=tree.meta.column, ) - result = celpy.celtypes.UintType(value_token.value[:-1]) + result_value = celpy.celtypes.UintType(value_token.value[:-1]) elif value_token.type in ("MLSTRING_LIT", "STRING_LIT"): - result = celstr(value_token) + result_value = celstr(value_token) elif value_token.type == "BYTES_LIT": - result = celbytes(value_token) + result_value = celbytes(value_token) elif value_token.type == "BOOL_LIT": - result = celpy.celtypes.BoolType(value_token.value.lower() == "true") + result_value = celpy.celtypes.BoolType( + value_token.value.lower() == "true" + ) elif value_token.type == "NULL_LIT": - result = None + result_value = None else: raise CELUnsupportedError( f"{tree.data} {tree.children}: type not implemented", @@ -2421,9 +2716,9 @@ def literal(self, tree: lark.Tree) -> Result: column=value_token.column or tree.meta.column, ) except ValueError as ex: - result = CELEvalError(ex.args[0], ex.__class__, ex.args, tree=tree) + result_value = CELEvalError(ex.args[0], ex.__class__, ex.args, tree=tree) - return result + return result_value @trace def exprlist(self, tree: lark.Tree) -> Result: @@ -2437,8 +2732,8 @@ def exprlist(self, tree: lark.Tree) -> Result: except StopIteration: pass # There are no CELEvalError values in the result, so we can narrow the domain. - result = celpy.celtypes.ListType(cast(List[celpy.celtypes.Value], values)) - return result + result_value = celpy.celtypes.ListType(cast(List[celpy.celtypes.Value], values)) + return result_value @trace def fieldinits(self, tree: lark.Tree) -> Result: @@ -2472,22 +2767,997 @@ def mapinits(self, tree: lark.Tree) -> Result: Extract the key expr's and value expr's to a list of pairs. This raises an exception on a duplicate key. - TODO: Is ``{'a': 1, 'b': 2/0}['a']`` a meaningful result in CEL? - Or is this an error because the entire member is erroneous? + .. TODO:: CEL question. + Is ``{'a': 1, 'b': 2/0}['a']`` a meaningful result in CEL? + Or is this an error because the entire member object is erroneous? + + .. TODO:: Refactor to use MapType([(key, value),...]) init, which checks for duplicates. + + This simplifies to ``celpy.celtypes.MapType(pairs)`` """ - result = celpy.celtypes.MapType() + result_value = celpy.celtypes.MapType() # Not sure if this cast is sensible. Should a CELEvalError propagate up from the # sub-expressions? See the error check in :py:func:`exprlist`. keys_values = cast(List[celpy.celtypes.Value], self.visit_children(tree)) pairs = zip(keys_values[0::2], keys_values[1::2]) for key, value in pairs: - if key in result: + if key in result_value: raise ValueError(f"Duplicate key {key!r}") - result[key] = value + result_value[key] = value + + return result_value + + +# GLOBAL activation used by Transpiled code. +# This slightly simplifies the exception handling, by using a 1-argument function +# to compute a Value or a CELEvalError. + +the_activation: Activation + + +def result(activation: Activation, cel_expr: Callable[[Activation], Result]) -> Result: + """ + Implements "checked exception" handling for CEL expressions transpiled to Python. + An expression must be wrapped by a lambda. + The lambda is evaluated by this function; a subset of Python exceptions become ``CELEvalError`` objects. + + >>> some_activation = Activation() + + Within the CEL transpiled code, we can now use code like this... + + >>> expr = lambda activation: 355 / 0 + >>> result(some_activation, expr) + CELEvalError(*('divide by zero', , ('division by zero',))) + + The exception becomes an object. + """ + + value: Result + try: + value = cel_expr(activation) + except ( + ValueError, + KeyError, + TypeError, + ZeroDivisionError, + OverflowError, + IndexError, + NameError, + ) as ex: + ex_message = { + ValueError: "return error for overflow", + KeyError: f"no such member in mapping: {ex.args[0]!r}", + TypeError: "no such overload", + ZeroDivisionError: "divide by zero", + OverflowError: "return error for overflow", + IndexError: "invalid_argument", + UnicodeDecodeError: "invalid UTF-8", + NameError: f"undeclared reference to {ex.args[0]!r} (in container {ex.args[1:]!r})", + }[ex.__class__] + _, _, tb = sys.exc_info() + value = CELEvalError(ex_message, ex.__class__, ex.args).with_traceback(tb) + value.__cause__ = ex + logger.debug("result = %r", value) + return value + + +def macro_map( + activation: Activation, + bind_variable: str, + cel_expr: Callable[[Activation], celpy.celtypes.Value], + cel_gen: Callable[[Activation], Iterable[Activation]], +) -> Result: + """The results of a source.map(v, expr) macro: a list of values.""" + activations = ( + activation.nested_activation(vars={bind_variable: cast(Result, _value)}) + for _value in cel_gen(activation) + ) + return celpy.celtypes.ListType(map(cel_expr, activations)) + + +def macro_filter( + activation: Activation, + bind_variable: str, + cel_expr: Callable[[Activation], celpy.celtypes.Value], + cel_gen: Callable[[Activation], Iterable[Activation]], +) -> Result: + """The results of a source.filter(v, expr) macro: a list of values.""" + r: list[celpy.celtypes.Value] = [] + for value in cel_gen(activation): + f = cel_expr( + activation.nested_activation(vars={bind_variable: cast(Result, value)}) + ) + if bool(f): + r.append(cast(celpy.celtypes.Value, value)) + return celpy.celtypes.ListType(iter(r)) + + +def macro_exists_one( + activation: Activation, + bind_variable: str, + cel_expr: Callable[[Activation], celpy.celtypes.Value], + cel_gen: Callable[[Activation], Iterable[Activation]], +) -> Result: + """The results of a source.exists_one(v, expr) macro: a list of values. + + Note the short-circuit concept. + Count the True; Break on an Exception + """ + count = 0 + activations = ( + activation.nested_activation(vars={bind_variable: cast(Result, _value)}) + for _value in cel_gen(activation) + ) + for result in filter(cel_expr, activations): + count += 1 if bool(result) else 0 + return celpy.celtypes.BoolType(count == 1) + + +def macro_exists( + activation: Activation, + bind_variable: str, + cel_expr: Callable[[Activation], celpy.celtypes.Value], + cel_gen: Callable[[Activation], Iterable[Activation]], +) -> Result: + """The results of a source.exists(v, expr) macro: a list of values.""" + activations = ( + activation.nested_activation(vars={bind_variable: cast(Result, _value)}) + for _value in cel_gen(activation) + ) + return celpy.celtypes.BoolType( + reduce( + cast( + Callable[[celpy.celtypes.BoolType, Result], celpy.celtypes.BoolType], + celpy.celtypes.logical_or, + ), + (result(act, cel_expr) for act in activations), + celpy.celtypes.BoolType(False), + ) + ) + + +def macro_all( + activation: Activation, + bind_variable: str, + cel_expr: Callable[[Activation], celpy.celtypes.Value], + cel_gen: Callable[[Activation], Iterable[Activation]], +) -> Result: + """The results of a source.all(v, expr) macro: a list of values.""" + activations = ( + activation.nested_activation(vars={bind_variable: cast(Result, _value)}) + for _value in cel_gen(activation) + ) + return celpy.celtypes.BoolType( + reduce( + cast( + Callable[[celpy.celtypes.BoolType, Result], celpy.celtypes.BoolType], + celpy.celtypes.logical_and, + ), + (result(act, cel_expr) for act in activations), + celpy.celtypes.BoolType(True), + ) + ) + + +class TranspilerTree(lark.Tree): + data: str + children: "Sequence[Union[lark.Token, TranspilerTree]]" # type: ignore[assignment] + + def __init__( + self, + data: str, + children: "Sequence[Union[lark.Token, TranspilerTree]]", + meta: Optional[lark.tree.Meta] = None, + ) -> None: + super().__init__(data, children, meta) # type: ignore [arg-type] + self.expr_number: int = 0 # Updated by visitor + self.transpiled: str = ( + f"ex_{self.expr_number}(activation)" # Default, often replaced. + ) + self.checked_exception: Union[ + tuple[Template, dict[str, Callable[[TranspilerTree], str]]], None + ] = None # Optional + + +class Transpiler: + """ + Transpile the CEL construct(s) to Python functions. + This is a **Facade** that wraps two visitor subclasses to do two phases + of transpilation. + + The resulting Python code can be used with ``compile()`` and ``exec()``. + + :Phase I: + The easy transpilation. + It builds simple text expressions for each node of the AST. + This sets aside exception-checking code including short-circuit logic operators and macros. + This decorates the AST with transpiled Python where possible. + It can also decorate with ``Template`` objects that require text from children. + + :Phase II: + Collects a sequence of statements. + All of the exception-checking for short-circuit logic operators and macros is packaged as lambdas + that may (or may not) be evaluated. + + Ideally, there could be a ``Transpiler`` ABC, and the ``PythonTranspiler`` defined as a subclass. + Pragmatically, we can't see any other sensible transpilation. + + .. rubric:: Exception Checking + + To handle ``2 / 0 || true``, the ``||``, ``&&``, and ``?:`` operators + the generated code creates lambdas to avoid execution where possible. + An alternative is a Monad-like structure to bottle up an exception, silencing it if it's unused. + + .. rubric:: Identifiers + + Identifiers have three meanings: + + - An object. This is either a variable provided in an ``Activation`` or a function provided + when building an execution. Objects also have type annotations. + + - A type annotation without an object. This can used to build protobuf messages. + + - A macro name. The ``member_dot_arg`` construct (e.g., ``member.map(v, expr)``) may have a macro instead of a method. + Plus the ``ident_arg`` construct may be a ``dyn()`` or ``has()`` macro instead of a function + + Other than macros, a name maps to an ``Referent`` instance. + This will have an annotation and -- perhaps -- an associated object. + + .. important MACROS ARE SPECIAL + + They aren't simple functions. + + The macros do not simply visit their children to perform evaluation. + + There's a bind variable and a function with the bind variable. + This isn't **trivially** moved from expression stack to statements. + + There are two functions that are macro-like: + + - ``dyn()`` does effectively nothing. + It visits its children, but also provides progressive type resolution + through detailed type annotation of the AST. + + - ``has()`` attempts to visit the child and does a boolean transformation + on the resulting exception or value. + This is a macro because it doesn't raise the exception for a missing + member item reference, but instead maps any exception to ``False``. + It doesn't return the value found for a member item reference; instead, it maps + successfully finding a member to ``True``. + + The member and expression list of a macro are transformed into lambdas for use by + special ``macro_{name}`` functions. These functions provided the necessary generator + expression to provide CEL semantics. + + Names have nested paths. For example, ``a.b.c`` is a mapping ``a``, that contains a mapping, ``b``, + that contains a name ``c``. + + The :py:meth:`member` method implements the macro evaluation behavior. + It does not **always** trivially descend into the children. + In the case of macros, the member evaluates one child tree in the presence + of values from another child tree using specific variable binding in a kind + of stack frame. + """ + + logger = logging.getLogger("celpy.Transpiler") + + def __init__( + self, + ast: TranspilerTree, + activation: Activation, + ) -> None: + """ + Create the Transpiler for an AST with specific variables and functions. + + :param ast: The AST to transpile. + :param activation: An activation with functions and types to use. + """ + self.ast = ast + self.base_activation = activation + self.activation = self.base_activation + + self.logger.debug("Transpiler activation: %r", self.activation) + # self.logger.debug("functions: %r", self.functions) # Refactor ``self.functions`` into an Activation + + def transpile(self) -> None: + """Two-phase transpilation. + + 1. Decorate AST with the most constructs. + 2. Expand into statements for lambdas that wrap checked exceptions. + """ + phase_1 = Phase1Transpiler(self) + phase_1.visit(self.ast) + + phase_2 = Phase2Transpiler(self) + phase_2.visit(self.ast) + + statements = phase_2.statements(self.ast) + + # The complete sequence of statements and the code object. + self.source_text = "\n".join(statements) + self.executable_code = compile(self.source_text, "", "exec") + + def evaluate(self, context: Context) -> celpy.celtypes.Value: + if context: + self.activation = self.base_activation.clone() + self.activation.identifiers.load_values(context) + else: + self.activation = self.base_activation + self.logger.debug("Activation: %r", self.activation) + + # Global for the top-level ``CEL = result(base_activation, ...)`` statement. + evaluation_globals = ( + celpy.evaluation.result.__globals__ + ) # the ``evaluation`` moodule + evaluation_globals["base_activation"] = self.activation + try: + exec(self.executable_code, evaluation_globals) + value = cast(celpy.celtypes.Value, evaluation_globals["CEL"]) + if isinstance(value, CELEvalError): + raise value + return value + except Exception as ex: + # A Python problem during ``exec()`` + self.logger.error("Internal error: %r", ex) + raise CELEvalError("evaluation error", type(ex), ex.args) + + +class Phase1Transpiler(lark.visitors.Visitor_Recursive): + """ + Decorate all nodes with transpiled Python code, where possible. + For short-circuit operators or macros, where a "checked exception" is required, + a simple ``ex_{n}`` name is present, and separate statements are provided as + a decoration to handle the more complicated cases. + + Each construct has an associated ``Template``. + For the simple cases, the transpiled value is the entire expression. + + >>> from unittest.mock import Mock + >>> source = "7 * (3 + 3)" + >>> parser = celpy.CELParser() + >>> tree = parser.parse(source) + >>> tp = Phase1Transpiler(Mock(base_activation=celpy.Activation())) + >>> _ = tp.visit(tree) + >>> tree.transpiled + 'operator.mul(celpy.celtypes.IntType(7), operator.add(celpy.celtypes.IntType(3), celpy.celtypes.IntType(3)))' + + Some constructs wrap macros or short-circuit logic, and require a more sophisticated execution. + There will be "checked exceptions", returned as values. + This requires statements with lambdas that can be wrapped by the ``result()`` function. + + The ``Phase2Transpiler`` does this transformation from expressions to a sequence of statements. + """ + + def __init__(self, facade: Transpiler) -> None: + self.facade = facade + self.activation = facade.base_activation + self.expr_number = 0 + + def visit(self, tree: TranspilerTree) -> TranspilerTree: # type: ignore[override] + """Initialize the decorations for each node.""" + tree.expr_number = self.expr_number + # tree.transpiled = f"ex_{tree.expr_number}(activation)" # Default, will be replaced. + # tree.checked_exception: Union[str, None] = None # Optional + self.expr_number += 1 + return super().visit(tree) # type: ignore[return-value] + + def func_name(self, label: str) -> str: + """ + Provide a transpiler-friendly name for the function. + + Some internally-defined functions appear to come from ``_operator`` module. + We need to rename some ``celpy`` functions to be from ``operator``. - return result + Some functions -- specifically lt, le, gt, ge, eq, ne -- are wrapped ``boolean(operator.f)`` + obscuring their name. + """ + try: + # func = self.functions[label] # Refactor ``self.functions`` into an Activation + func = self.activation.resolve_function(label) + except KeyError: + return f"CELEvalError('unbound function', KeyError, ({label!r},))" + module = {"_operator": "operator"}.get(func.__module__, func.__module__) + return f"{module}.{func.__qualname__}" + + def expr(self, tree: TranspilerTree) -> None: + """ + expr : conditionalor ["?" conditionalor ":" expr] + """ + if len(tree.children) == 1: + tree.transpiled = cast(TranspilerTree, tree.children[0]).transpiled + elif len(tree.children) == 3: + template = Template( + dedent("""\ + # expr: + ex_${n}_c = lambda activation: ${cond} + ex_${n}_l = lambda activation: ${left} + ex_${n}_r = lambda activation: ${rght} + ex_${n} = lambda activation: ${func_name}(celpy.evaluation.result(activation, ex_${n}_c), celpy.evaluation.result(activation, ex_${n}_l), celpy.evaluation.result(activation, ex_${n}_r))""") + ) + tree.checked_exception = ( + template, + dict( + n=lambda tree: str(tree.expr_number), + func_name=lambda tree: self.func_name("_?_:_"), + cond=lambda tree: cast(TranspilerTree, tree.children[0]).transpiled, + left=lambda tree: cast(TranspilerTree, tree.children[1]).transpiled, + rght=lambda tree: cast(TranspilerTree, tree.children[2]).transpiled, + ), + ) + tree.transpiled = f"ex_{tree.expr_number}(activation)" + + def conditionalor(self, tree: TranspilerTree) -> None: + """ + conditionalor : [conditionalor "||"] conditionaland + """ + if len(tree.children) == 1: + tree.transpiled = cast(TranspilerTree, tree.children[0]).transpiled + elif len(tree.children) == 2: + template = Template( + dedent("""\ + # conditionalor: + ex_${n}_l = lambda activation: ${left} + ex_${n}_r = lambda activation: ${rght} + ex_${n} = lambda activation: ${func_name}(celpy.evaluation.result(activation, ex_${n}_l), celpy.evaluation.result(activation, ex_${n}_r))""") + ) + tree.checked_exception = ( + template, + dict( + n=lambda tree: str(tree.expr_number), + func_name=lambda tree: self.func_name("_||_"), + left=lambda tree: cast(TranspilerTree, tree.children[0]).transpiled, + rght=lambda tree: cast(TranspilerTree, tree.children[1]).transpiled, + ), + ) + tree.transpiled = f"ex_{tree.expr_number}(activation)" + + def conditionaland(self, tree: TranspilerTree) -> None: + """ + conditionaland : [conditionaland "&&"] relation + """ + if len(tree.children) == 1: + tree.transpiled = cast(TranspilerTree, tree.children[0]).transpiled + elif len(tree.children) == 2: + template = Template( + dedent("""\ + # conditionaland: + ex_${n}_l = lambda activation: ${left} + ex_${n}_r = lambda activation: ${rght} + ex_${n} = lambda activation: ${func_name}(celpy.evaluation.result(activation, ex_${n}_l), celpy.evaluation.result(activation, ex_${n}_r))""") + ) + tree.checked_exception = ( + template, + dict( + n=lambda tree: str(tree.expr_number), + func_name=lambda tree: self.func_name("_&&_"), + left=lambda tree: cast(TranspilerTree, tree.children[0]).transpiled, + rght=lambda tree: cast(TranspilerTree, tree.children[1]).transpiled, + ), + ) + tree.transpiled = f"ex_{tree.expr_number}(activation)" + + def relation(self, tree: TranspilerTree) -> None: + """ + relation : [relation_lt | relation_le | relation_ge | relation_gt + | relation_eq | relation_ne | relation_in] addition + + relation_lt : relation "<" + relation_le : relation "<=" + relation_gt : relation ">" + relation_ge : relation ">=" + relation_eq : relation "==" + relation_ne : relation "!=" + relation_in : relation "in" + """ + if len(tree.children) == 1: + tree.transpiled = cast(TranspilerTree, tree.children[0]).transpiled + elif len(tree.children) == 2: + left_op, right_tree = cast( + Tuple[TranspilerTree, TranspilerTree], tree.children + ) + op_name = { + "relation_lt": "_<_", + "relation_le": "_<=_", + "relation_ge": "_>=_", + "relation_gt": "_>_", + "relation_eq": "_==_", + "relation_ne": "_!=_", + "relation_in": "_in_", + }[left_op.data] + func_name = self.func_name(op_name) + template = Template("${func_name}(${left}, ${right})") + tree.transpiled = template.substitute( + func_name=func_name, + left=cast(TranspilerTree, left_op.children[0]).transpiled, + right=right_tree.transpiled, + ) + + def addition(self, tree: TranspilerTree) -> None: + """ + addition : [addition_add | addition_sub] multiplication + + addition_add : addition "+" + addition_sub : addition "-" + """ + if len(tree.children) == 1: + tree.transpiled = cast(TranspilerTree, tree.children[0]).transpiled + elif len(tree.children) == 2: + left_op, right_tree = cast( + Tuple[TranspilerTree, TranspilerTree], tree.children + ) + op_name = { + "addition_add": "_+_", + "addition_sub": "_-_", + }[left_op.data] + func_name = self.func_name(op_name) + template = Template("${func_name}(${left}, ${right})") + tree.transpiled = template.substitute( + func_name=func_name, + left=cast(TranspilerTree, left_op.children[0]).transpiled, + right=right_tree.transpiled, + ) + + def multiplication(self, tree: TranspilerTree) -> None: + """ + multiplication : [multiplication_mul | multiplication_div | multiplication_mod] unary + + multiplication_mul : multiplication "*" + multiplication_div : multiplication "/" + multiplication_mod : multiplication "%" + """ + if len(tree.children) == 1: + tree.transpiled = cast(TranspilerTree, tree.children[0]).transpiled + elif len(tree.children) == 2: + template = Template("${func_name}(*${children})") + left_op, right_tree = cast( + Tuple[TranspilerTree, TranspilerTree], tree.children + ) + op_name = { + "multiplication_mul": "_*_", + "multiplication_div": "_/_", + "multiplication_mod": "_%_", + }[left_op.data] + func_name = self.func_name(op_name) + template = Template("${func_name}(${left}, ${right})") + tree.transpiled = template.substitute( + func_name=func_name, + left=cast(TranspilerTree, left_op.children[0]).transpiled, + right=right_tree.transpiled, + ) + + def unary(self, tree: TranspilerTree) -> None: + """ + unary : [unary_not | unary_neg] member + + unary_not : "!" + unary_neg : "-" + """ + if len(tree.children) == 1: + tree.transpiled = cast(TranspilerTree, tree.children[0]).transpiled + elif len(tree.children) == 2: + template = Template("${func_name}(${children})") + op_tree, right_tree = cast( + Tuple[TranspilerTree, TranspilerTree], tree.children + ) + op_name = { + "unary_not": "!_", + "unary_neg": "-_", + }[op_tree.data] + func_name = self.func_name(op_name) + children = right_tree.transpiled + tree.transpiled = template.substitute( + func_name=func_name, children=children + ) + + def member(self, tree: TranspilerTree) -> None: + """ + member : member_dot | member_dot_arg | member_item | member_object | primary + """ + tree.transpiled = cast(TranspilerTree, tree.children[0]).transpiled + + def member_dot(self, tree: TranspilerTree) -> None: + """ + member_dot : member "." IDENT + + .. important:: + + The ``member`` can be any of a variety of objects: + + - ``NameContainer(Dict[str, Referent])`` + + - ``Activation`` + + - ``MapType(Dict[Value, Value])`` + + - ``MessageType(MapType)`` + + All of which define a ``get()`` method. + The nuance is the ``NameContainer`` is also a Python ``dict`` and there's an + overload issue between that ``get()`` and other ``get()`` definitions. + + .. todo:: Define a new get_name(member, 'name') function do this, avoiding the ``get()`` method. + """ + member_tree, property_name_token = cast( + Tuple[TranspilerTree, lark.Token], tree.children + ) + template = Template("${left}.get('${right}')") + tree.transpiled = template.substitute( + left=member_tree.transpiled, right=property_name_token.value + ) + + def member_dot_arg(self, tree: TranspilerTree) -> None: + """ + member_dot_arg : member "." IDENT "(" [exprlist] ")" + + Two flavors: macro and non-macro. + """ + exprlist: Union[TranspilerTree, None] + if len(tree.children) == 3: + member_tree, property_name_token, exprlist = cast( + Tuple[TranspilerTree, lark.Token, TranspilerTree], tree.children + ) + else: + # len(tree.children) == 2, no [exprlist]. + member_tree, property_name_token = cast( + Tuple[TranspilerTree, lark.Token], tree.children + ) + exprlist = None + if property_name_token.value in { + "map", + "filter", + "all", + "exists", + "exists_one", + "reduce", + "min", + }: + # Macro. Defer to Phase II. + template = Template( + dedent("""\ + # member_dot_arg ${macro}: + ex_${n}_l = lambda activation: ${member} + ex_${n}_x = lambda activation: ${expr} + ex_${n} = lambda activation: celpy.evaluation.macro_${macro}(activation, '${bind_variable}', ex_${n}_x, ex_${n}_l) + """) + ) + if len(tree.children) == 3: + # Hackery. Undo the transpiling of the identifier and extract only the name. + context, bind_variable = cast( + TranspilerTree, cast(TranspilerTree, tree.children[2]).children[0] + ).transpiled.split(".") + else: + raise CELSyntaxError( # pragma: no cover + "no bind variable in {property_name_token.value} macro", + line=tree.meta.line, + column=tree.meta.column, + ) + + tree.checked_exception = ( + template, + dict( + n=lambda tree: str(tree.expr_number), + macro=lambda tree: property_name_token.value, + member=lambda tree: cast( + TranspilerTree, tree.children[0] + ).transpiled, + bind_variable=lambda tree: bind_variable, + expr=lambda tree: cast( + TranspilerTree, + cast(TranspilerTree, tree.children[2]).children[1], + ).transpiled + if len(tree.children) == 3 + else "", + ), + ) + tree.transpiled = f"ex_{tree.expr_number}(activation)" + + else: + # Non-macro method name. + if exprlist: + template = Template("${func_name}(${left}, ${right})") + func_name = self.func_name(property_name_token.value) + tree.transpiled = template.substitute( + func_name=func_name, + left=member_tree.transpiled, + right=exprlist.transpiled, + ) + else: + template = Template("${func_name}(${left})") + func_name = self.func_name(property_name_token.value) + tree.transpiled = template.substitute( + func_name=func_name, + left=member_tree.transpiled, + ) + + def member_index(self, tree: TranspilerTree) -> None: + """ + member_item : member "[" expr "]" + """ + template = Template("${func_name}(${member}, ${expr})") + member, expr = cast(tuple[TranspilerTree, TranspilerTree], tree.children) + func_name = self.func_name("_[_]") + tree.transpiled = template.substitute( + func_name=func_name, + member=member.transpiled, + expr=expr.transpiled, + ) + + def member_object(self, tree: TranspilerTree) -> None: + """ + member_object : member "{" [fieldinits] "}" + """ + template = Template("${type_name}([${fieldinits}])") + member = cast(TranspilerTree, tree.children[0]) + fieldinits: str + if len(tree.children) == 2: + fieldinits = cast(TranspilerTree, tree.children[1]).transpiled + else: + fieldinits = "" + type_name = member.transpiled + tree.transpiled = template.substitute( + type_name=type_name, fieldinits=fieldinits + ) + + def primary(self, tree: TranspilerTree) -> None: + """ + primary : dot_ident_arg | dot_ident | ident_arg | ident + | paren_expr | list_lit | map_lit | literal + """ + tree.transpiled = cast(TranspilerTree, tree.children[0]).transpiled + + def dot_ident_arg(self, tree: TranspilerTree) -> None: + """ + dot_ident_arg : "." IDENT "(" [exprlist] ")" + """ + template = Template("activation.resolve_variable('${ident}')(${exprlist})") + ident = cast(lark.Token, tree.children[0]).value + if len(tree.children) == 2: + exprlist = cast(TranspilerTree, tree.children[1]).transpiled + else: + exprlist = "" + tree.transpiled = template.substitute(ident=ident, exprlist=exprlist) + + def dot_ident(self, tree: TranspilerTree) -> None: + """ + dot_ident : "." IDENT + """ + template = Template("activation.resolve_variable('${ident}')") + ident = cast(lark.Token, tree.children[0]).value + tree.transpiled = template.substitute(ident=ident) + + def ident_arg(self, tree: TranspilerTree) -> None: + """ + ident_arg : IDENT "(" [exprlist] ")" + """ + op = cast(lark.Token, tree.children[0]).value + if len(tree.children) == 2: + exprlist = cast(TranspilerTree, tree.children[1]).transpiled + else: + exprlist = "" + if op in {"has", "dyn"}: + # Macro-like has() or dyn() + if op == "dyn": + tree.transpiled = cast(TranspilerTree, tree.children[1]).transpiled + elif op == "has": + # try to evaluate the exprlist expression + # TODO: as macro_has() would be better... + template = Template( + dedent("""\ + # ident_arg has: + ex_${n}_h = lambda activation: ${exprlist} + ex_${n} = lambda activation: not isinstance(celpy.evaluation.result(activation, ex_${n}_h), CELEvalError) + """) + ) + tree.checked_exception = ( + template, + dict( + n=lambda tree: str(tree.expr_number), + exprlist=lambda tree: cast( + TranspilerTree, tree.children[1] + ).transpiled, + ), + ) + tree.transpiled = f"ex_{tree.expr_number}(activation)" + else: + # Other function + template = Template("${func_name}(${exprlist})") + func_name = self.func_name(op) + if len(tree.children) == 2: + exprlist = cast(TranspilerTree, tree.children[1]).transpiled + else: + exprlist = "" + tree.transpiled = template.substitute( + func_name=func_name, exprlist=exprlist + ) + + def ident(self, tree: TranspilerTree) -> None: + """ + ident : IDENT + """ + template = Template("activation.${ident}") + tree.transpiled = template.substitute( + ident=cast(lark.Token, tree.children[0]).value + ) + + def paren_expr(self, tree: TranspilerTree) -> None: + """ + paren_expr : "(" expr ")" + """ + tree.transpiled = cast(Sequence[TranspilerTree], tree.children)[0].transpiled + + def list_lit(self, tree: TranspilerTree) -> None: + """ + list_lit : "[" [exprlist] "]" + """ + if tree.children: + exprlist = cast(Sequence[TranspilerTree], tree.children)[0].transpiled + else: + exprlist = "" + template = Template("celpy.celtypes.ListType([${exprlist}])") + tree.transpiled = template.substitute(exprlist=exprlist) + + def map_lit(self, tree: TranspilerTree) -> None: + """ + map_lit : "{" [mapinits] "}" + """ + if tree.children: + mapinits = cast(Sequence[TranspilerTree], tree.children)[0].transpiled + else: + mapinits = "" + template = Template("celpy.celtypes.MapType([${mapinits}])") + tree.transpiled = template.substitute(mapinits=mapinits) + + def exprlist(self, tree: TranspilerTree) -> None: + """ + exprlist : expr ("," expr)* + """ + exprs = ", ".join( + c.transpiled for c in cast(Sequence[TranspilerTree], tree.children) + ) + tree.transpiled = exprs + + def fieldinits(self, tree: TranspilerTree) -> None: + """ + fieldinits : IDENT ":" expr ("," IDENT ":" expr)* + """ + idents = [ + cast(lark.Token, c).value + for c in cast(Sequence[TranspilerTree], tree.children)[::2] + ] + exprs = [ + c.transpiled for c in cast(Sequence[TranspilerTree], tree.children)[1::2] + ] + assert len(idents) == len(exprs), "Invalid AST" + items = ", ".join(f"('{n}', {v})" for n, v in zip(idents, exprs)) + tree.transpiled = items + + def mapinits(self, tree: TranspilerTree) -> None: + """ + mapinits : expr ":" expr ("," expr ":" expr)* + """ + keys = [ + c.transpiled for c in cast(Sequence[TranspilerTree], tree.children)[::2] + ] + values = [ + c.transpiled for c in cast(Sequence[TranspilerTree], tree.children)[1::2] + ] + assert len(keys) == len(values) + items = ", ".join(f"({k}, {v})" for k, v in zip(keys, values)) + tree.transpiled = items + + def literal(self, tree: TranspilerTree) -> None: + """ + literal : UINT_LIT | FLOAT_LIT | INT_LIT | MLSTRING_LIT | STRING_LIT | BYTES_LIT + | BOOL_LIT | NULL_LIT + """ + value_token = cast(lark.Token, tree.children[0]) + if value_token.type == "FLOAT_LIT": + lit_text = f"celpy.celtypes.DoubleType({value_token.value})" + elif value_token.type == "INT_LIT": + lit_text = f"celpy.celtypes.IntType({value_token.value})" + elif value_token.type == "UINT_LIT": + if not value_token.value[-1].lower() == "u": + raise CELSyntaxError( + f"invalid unsigned int literal {value_token!r}", + line=tree.meta.line, + column=tree.meta.column, + ) # pragma: no cover + lit_text = f"celpy.celtypes.UintType({value_token.value[:-1]})" + elif value_token.type in ("MLSTRING_LIT", "STRING_LIT"): + lit_text = f"celpy.celtypes.{celstr(value_token)!r}" + elif value_token.type == "BYTES_LIT": + lit_text = f"celpy.celtypes.{celbytes(value_token)!r}" + elif value_token.type == "BOOL_LIT": + lit_text = f"celpy.celtypes.BoolType({value_token.value.lower() == 'true'})" + elif value_token.type == "NULL_LIT": + lit_text = "None" # Not celpy.celtypes.NullType() in transpiled code. + else: + raise CELUnsupportedError( + f"{tree.data} {tree.children}: type not implemented", + line=value_token.line or tree.meta.line, + column=value_token.column or tree.meta.column, + ) # pragma: no cover + tree.transpiled = lit_text + + +class Phase2Transpiler(lark.visitors.Visitor_Recursive): + """ + Extract any checked_exception evaluation statements that decorate the parse tree. + Also, get the overall top-level expression, assigned to special variable, CEL. + + >>> from unittest.mock import Mock + >>> from pprint import pprint + >>> source = '["hello", "world"].map(x, x) == ["hello", "world"]' + >>> celpy.CELParser.CEL_PARSER = None + >>> parser = celpy.CELParser(tree_class=celpy.evaluation.TranspilerTree) + >>> tree = parser.parse(source) + >>> tp1 = Phase1Transpiler(Mock(base_activation=celpy.Activation())) + >>> _ = tp1.visit(tree) + >>> tp2 = Phase2Transpiler(Mock(base_activation=celpy.Activation())) + >>> _ = tp2.visit(tree) + >>> pprint(tp2.statements(tree), width=256) + ['# member_dot_arg map:', + "ex_10_l = lambda activation: celpy.celtypes.ListType([celpy.celtypes.StringType('hello'), celpy.celtypes.StringType('world')])", + 'ex_10_x = lambda activation: activation.x', + "ex_10 = lambda activation: celpy.evaluation.macro_map(activation, 'x', ex_10_x, ex_10_l)", + "CEL = celpy.evaluation.result(base_activation, lambda activation: celpy.evaluation.bool_eq(ex_10(activation), celpy.celtypes.ListType([celpy.celtypes.StringType('hello'), celpy.celtypes.StringType('world')])))\\n"] + + """ + + def __init__(self, facade: Transpiler) -> None: + self.facade = facade + self._statements: list[str] = [] + + def expr(self, tree: TranspilerTree) -> None: + """ + expr : conditionalor ["?" conditionalor ":" expr] + + All checked_exception structures are a tuple[Template, dict[str, Callable[[Tree], str]] + """ + if tree.checked_exception: + template, bindings = tree.checked_exception + self._statements.extend( + template.substitute( + {k: v(tree) for k, v in bindings.items()} + ).splitlines() + ) + + conditionalor = expr + conditionaland = expr + member_dot_arg = expr + ident_arg = expr + + def statements(self, tree: TranspilerTree) -> list[str]: + """ + Appends the final CEL = ... statement to the sequence of statements, + and returns the transpiled code. + + Two patterns: + + 1. Top-most expr was a deferred template, and already is a lambda. + It will be a string of the form ``"ex_\\d+(activation)"``. + + 2. Top-most expr was **not** a deferred template, and needs a lambda wrapper. + It will **not** be a simple ``"ex_\\d+"`` reference. + """ + expr_pattern = re.compile(r"^(ex_\d+)\(\w+\)$") + if match := expr_pattern.match(tree.transpiled): + template = Template( + dedent("""\ + CEL = celpy.evaluation.result(base_activation, ${lambda_name}) + """) + ) + final = template.substitute(n=tree.expr_number, lambda_name=match.group(1)) + else: + template = Template( + dedent("""\ + CEL = celpy.evaluation.result(base_activation, lambda activation: ${expr}) + """) + ) + final = template.substitute(n=tree.expr_number, expr=tree.transpiled) + return self._statements + [final] CEL_ESCAPES_PAT = re.compile( diff --git a/tests/test_c7nlib.py b/tests/test_c7nlib.py index f155c88..6371685 100644 --- a/tests/test_c7nlib.py +++ b/tests/test_c7nlib.py @@ -29,7 +29,7 @@ from types import SimpleNamespace from unittest.mock import Mock, call, sentinel -from pytest import * +import pytest import celpy import celpy.adapter @@ -167,7 +167,7 @@ def test_version(): ), ] -@fixture(params=value_from_examples) +@pytest.fixture(params=value_from_examples) def mock_urllib_request(monkeypatch, request): suffix, encoding, raw_bytes, expected = request.param urllib_request = Mock( @@ -197,7 +197,7 @@ def test_value_from(mock_urllib_request): def test_value_from_bad_format(): - with raises(ValueError): + with pytest.raises(ValueError): celpy.c7nlib.value_from(sentinel.URL, format="nope") @@ -207,7 +207,7 @@ def test_value_from_bad_format(): ({"foo": {"bar": [{"name": "one"}, {"name": "two"}]}}, "foo.bar[*].name", ["one", "two"]), ] -@fixture(params=jmes_path_examples) +@pytest.fixture(params=jmes_path_examples) def doc_path_expected(request): json_doc, path, expected = request.param return ( @@ -223,7 +223,7 @@ def test_jmes_path(doc_path_expected): assert expected == actual -@fixture(params=jmes_path_examples) +@pytest.fixture(params=jmes_path_examples) def doclist_path_expected(request): json_doc, path, expected = request.param return ( @@ -335,17 +335,12 @@ def test_arn_split(): assert celpy.c7nlib.arn_split(f3, "resource-type") == "resource-type-3" assert celpy.c7nlib.arn_split(f3, "resource-id") == "resource-id-3" - with raises(ValueError): + with pytest.raises(ValueError): celpy.c7nlib.arn_split("http://server.name:port/path/to/resource", "partition") -@fixture -def celfilter_instance(): - """ - The mocked CELFilter instance for all of the c7nlib integration tests. - - This CELFilter class demonstrates *all* the features required for the refactored C7N. - """ +@pytest.fixture +def mock_manager(): datapoints = [ {"Average": str(sentinel.average)} ] @@ -466,7 +461,7 @@ def celfilter_instance(): "rds": rds_resource_manager, "waf": waf_resource_manager, } - mock_manager = Mock( + manager = Mock( name="mock_manager", session_factory=Mock(return_value=mock_session), get_model=Mock(return_value=Mock(dimension="InstanceId", id="InstanceId", service="ec2")), @@ -479,7 +474,13 @@ def celfilter_instance(): ), data={"resource": "ec2"} ) + return locals() +@pytest.fixture +def mock_filter_class(): + """ + This CELFilter class demonstrates *all* the features required for the refactored C7N. + """ mock_parser = Mock( name="Mock c7n.filters.offhours.ScheduleParser instance", parse=Mock( @@ -583,12 +584,18 @@ def __init__(self, data, manager): assert self.data["type"].lower() == "cel" self.expr = self.data["expr"] self.parser = mock_parser + return CELFilter +@pytest.fixture +def celfilter_instance(mock_filter_class, mock_manager): + """ + The mocked CELFilter instance for all of the c7nlib integration tests. + """ # A place-holder used only for initialization. mock_policy_filter_source = {"type": "cel", "expr": "1+1==2"} # The mock for the ``CELFilter`` instance C7N must provide. - the_filter = CELFilter(mock_policy_filter_source, mock_manager) + the_filter = mock_filter_class(mock_policy_filter_source, mock_manager['manager']) return locals() @@ -603,7 +610,7 @@ def test_image_age_good(celfilter_instance): assert doc.get(celpy.celtypes.StringType('CreationDate')) == celpy.celtypes.TimestampType("2020-09-10T11:12:13Z") -def test_image_age_missing(celfilter_instance): +def test_image_age_missing(celfilter_instance, mock_manager): mock_filter = celfilter_instance['the_filter'] mock_filter.get_instance_image=Mock(return_value=None) resource = celpy.celtypes.MapType({}) @@ -614,9 +621,9 @@ def test_image_age_missing(celfilter_instance): assert doc.get(celpy.celtypes.StringType('CreationDate')) == celpy.celtypes.TimestampType("2000-01-01T01:01:01.000Z") -def test_get_raw_metrics(celfilter_instance): +def test_get_raw_metrics(celfilter_instance, mock_manager): mock_filter = celfilter_instance['the_filter'] - datapoints = celfilter_instance['datapoints'] + datapoints = mock_manager['datapoints'] now = celpy.celtypes.TimestampType("2000-01-01T01:01:01.000Z") resource = celpy.celtypes.MapType({}) request = celpy.celtypes.MapType( @@ -644,14 +651,14 @@ def test_get_raw_metrics(celfilter_instance): assert doc == celpy.json_to_cel(datapoints) -def test_get_metrics(celfilter_instance): +def test_get_metrics(celfilter_instance, mock_manager): """ Two approaches possible. (1) mock :func:`get_raw_metrics`. (2) provide mocks to support :func:`get_raw_metrics`. We use approach 2 in case the implmentation of `get_metrics` is changed. """ mock_filter = celfilter_instance['the_filter'] - datapoints = celfilter_instance['datapoints'] + datapoints = mock_manager['datapoints'] now = celpy.celtypes.TimestampType("2000-01-01T01:01:01.000Z") resource = celpy.celtypes.MapType({"InstanceId": "i-123456789012"}) @@ -983,6 +990,28 @@ def test_C7N_interpreted_runner(celfilter_instance): # Did it work? assert cel_result +def test_C7N_extension_function(mock_filter_class, mock_manager, caplog): + mock_policy_filter_source = {"type": "cel", "expr": '"PRE-this".glob("PRE-*")'} + + # The mock for the ``CELFilter`` instance C7N must provide. + the_filter = mock_filter_class(mock_policy_filter_source, + mock_manager['manager']) + + cel_env = celpy.Environment(runner_class=celpy.c7nlib.C7N_Interpreted_Runner) + cel_ast = cel_env.compile(the_filter.expr) + + # This will be implemented in ``CELFilter.process()`` or ``CELFilter.__call__()``. + cel_prgm = cel_env.program(cel_ast, functions=celpy.c7nlib.FUNCTIONS) + cel_activation = { + "resource": celpy.celtypes.MapType({}), + "now": celpy.celtypes.TimestampType("2020-09-10T11:12:13Z"), + } + with celpy.c7nlib.C7NContext(filter=Mock()): + cel_result = cel_prgm.evaluate(cel_activation, filter=the_filter) + + # Did it work? + assert cel_result + def test_C7N_CELFilter_image(celfilter_instance): mock_filter = celfilter_instance['the_filter'] @@ -994,7 +1023,7 @@ def test_C7N_CELFilter_image(celfilter_instance): assert mock_filter.get_instance_image.mock_calls == [call(ec2_doc)] -def test_C7N_CELFilter_get_raw_metrics(celfilter_instance): +def test_C7N_CELFilter_get_raw_metrics(celfilter_instance, mock_manager): mock_filter = celfilter_instance['the_filter'] metrics_doc = { "Namespace": "AWS/EC2", @@ -1018,12 +1047,12 @@ def test_C7N_CELFilter_get_raw_metrics(celfilter_instance): Period=metrics_doc["Period"], Dimensions=metrics_doc["Dimensions"], ) - cloudwatch_client = celfilter_instance['cloudwatch_client'] + cloudwatch_client = mock_manager['cloudwatch_client'] print(f"cloudwatch_client {cloudwatch_client}") assert cloudwatch_client.mock_calls == [call.get_metric_statistics(**expected_request)] -def test_C7N_CELFilter_get_metrics(celfilter_instance): +def test_C7N_CELFilter_get_metrics(celfilter_instance, mock_manager): mock_filter = celfilter_instance['the_filter'] ec2_doc = {"ResourceType": "ec2", "InstanceId": "i-123456789"} request = { @@ -1034,7 +1063,7 @@ def test_C7N_CELFilter_get_metrics(celfilter_instance): metrics = celpy.c7nlib.get_metrics(ec2_doc, request) assert metrics == [str(sentinel.average)] - cloudwatch_client = celfilter_instance['cloudwatch_client'] + cloudwatch_client = mock_manager['cloudwatch_client'] expected_request = dict( Namespace=celpy.celtypes.StringType("ec2"), MetricName=request["MetricName"], @@ -1134,7 +1163,7 @@ def test_C7N_CELFilter_subnet(celfilter_instance): assert mock_filter.get_related.mock_calls == [call([ec2_doc])] -def test_C7N_CELFilter_flow_logs(celfilter_instance): +def test_C7N_CELFilter_flow_logs(celfilter_instance, mock_manager): mock_filter = celfilter_instance['the_filter'] ec2_doc = {"ResourceType": "ec2", "InstanceId": "i-123456789"} with celpy.c7nlib.C7NContext(filter=mock_filter): @@ -1149,7 +1178,7 @@ def test_C7N_CELFilter_flow_logs(celfilter_instance): ) ] ) - ec2_client = celfilter_instance['ec2_client'] + ec2_client = mock_manager['ec2_client'] assert ec2_client.describe_flow_logs.mock_calls == [call()] def test_C7N_CELFilter_vpc(celfilter_instance): @@ -1226,7 +1255,7 @@ def test_C7N_CELFilter_all_snapshots(celfilter_instance): assert mock_filter._pull_ami_snapshots.mock_calls == [call()] -def test_C7N_CELFilter_all_launch_configuration_names(celfilter_instance): +def test_C7N_CELFilter_all_launch_configuration_names(celfilter_instance, mock_manager): mock_filter = celfilter_instance['the_filter'] asg_doc = {"ResourceType": "asg", "InstanceId": "i-123456789"} with celpy.c7nlib.C7NContext(filter=mock_filter): @@ -1235,7 +1264,7 @@ def test_C7N_CELFilter_all_launch_configuration_names(celfilter_instance): [celpy.celtypes.StringType(str(sentinel.asg_launch_config_name))] ) assert mock_filter.manager.get_resource_manager.mock_calls == [call('asg')] - assert celfilter_instance['asg_resource_manager'].resources.mock_calls == [call()] + assert mock_manager['asg_resource_manager'].resources.mock_calls == [call()] def test_C7N_CELFilter_all_service_roles(celfilter_instance): @@ -1260,7 +1289,7 @@ def test_C7N_CELFilter_all_instance_profiles(celfilter_instance): assert mock_filter.instance_profile_usage.mock_calls == [call()] -def test_C7N_CELFilter_all_dbsubenet_groups(celfilter_instance): +def test_C7N_CELFilter_all_dbsubenet_groups(celfilter_instance, mock_manager): mock_filter = celfilter_instance['the_filter'] rds_doc = {"ResourceType": "rds-subnet-group", "InstanceId": "i-123456789"} with celpy.c7nlib.C7NContext(filter=mock_filter): @@ -1269,7 +1298,7 @@ def test_C7N_CELFilter_all_dbsubenet_groups(celfilter_instance): [celpy.celtypes.StringType(str(sentinel.rds_subnet_group_name))] ) assert mock_filter.manager.get_resource_manager.mock_calls == [call('rds')] - assert celfilter_instance['rds_resource_manager'].resources.mock_calls == [call()] + assert mock_manager['rds_resource_manager'].resources.mock_calls == [call()] def test_C7N_CELFilter_all_scan_groups(celfilter_instance): @@ -1283,7 +1312,7 @@ def test_C7N_CELFilter_all_scan_groups(celfilter_instance): assert mock_filter.scan_groups.mock_calls == [call()] -def test_C7N_CELFilter_get_access_log(celfilter_instance): +def test_C7N_CELFilter_get_access_log(celfilter_instance, mock_manager): mock_filter = celfilter_instance['the_filter'] elb_doc = {"ResourceType": "elb", "LoadBalancerName": "i-123456789"} with celpy.c7nlib.C7NContext(filter=mock_filter): @@ -1296,11 +1325,11 @@ def test_C7N_CELFilter_get_access_log(celfilter_instance): assert mock_filter.manager.session_factory.return_value.client.mock_calls == [ call('elb') ] - assert celfilter_instance['elb_client'].describe_load_balancer_attributes.mock_calls == [ + assert mock_manager['elb_client'].describe_load_balancer_attributes.mock_calls == [ call(LoadBalancerName='i-123456789') ] -def test_C7N_CELFilter_get_load_balancer(celfilter_instance): +def test_C7N_CELFilter_get_load_balancer(celfilter_instance, mock_manager): mock_filter = celfilter_instance['the_filter'] elb_doc = {"ResourceType": "app-elb", "LoadBalancerArn": "arn:us-east-1:app-elb:123456789:etc"} with celpy.c7nlib.C7NContext(filter=mock_filter): @@ -1314,14 +1343,14 @@ def test_C7N_CELFilter_get_load_balancer(celfilter_instance): assert mock_filter.manager.session_factory.return_value.client.mock_calls == [ call('elbv2') ] - assert celfilter_instance['elbv2_client'].describe_load_balancer_attributes.mock_calls == [ + assert mock_manager['elbv2_client'].describe_load_balancer_attributes.mock_calls == [ call(LoadBalancerArn='arn:us-east-1:app-elb:123456789:etc') ] -def test_C7N_CELFilter_get_raw_health_events(celfilter_instance): +def test_C7N_CELFilter_get_raw_health_events(celfilter_instance, mock_manager): mock_filter = celfilter_instance['the_filter'] - health_events = celfilter_instance['health_events'] + health_events = mock_manager['health_events'] request = celpy.celtypes.MapType( { celpy.celtypes.StringType("services"): @@ -1343,9 +1372,9 @@ def test_C7N_CELFilter_get_raw_health_events(celfilter_instance): assert doc == celpy.json_to_cel(health_events) -def test_C7N_CELFilter_get_health_events(celfilter_instance): +def test_C7N_CELFilter_get_health_events(celfilter_instance, mock_manager): mock_filter = celfilter_instance['the_filter'] - health_events = celfilter_instance['health_events'] + health_events = mock_manager['health_events'] directory_doc = {"ResourceType": "directory", "arn": "arn:us-east-1:app-elb:123456789:etc"} with celpy.c7nlib.C7NContext(filter=mock_filter): health_events = celpy.c7nlib.get_health_events(directory_doc) @@ -1353,7 +1382,7 @@ def test_C7N_CELFilter_get_health_events(celfilter_instance): assert mock_filter.manager.session_factory.return_value.client.mock_calls == [ call('health', region_name='us-east-1') ] - assert celfilter_instance['health_client'].describe_events.mock_calls == [ + assert mock_manager['health_client'].describe_events.mock_calls == [ call( filter={ 'services': ['EC2'], @@ -1364,7 +1393,7 @@ def test_C7N_CELFilter_get_health_events(celfilter_instance): ] -def test_C7N_CELFilter_shield_protection(celfilter_instance): +def test_C7N_CELFilter_shield_protection(celfilter_instance, mock_manager): mock_filter = celfilter_instance['the_filter'] elb_doc = {"ResourceType": "elb", "arn": "arn:us-east-1:app-elb:123456789:etc"} with celpy.c7nlib.C7NContext(filter=mock_filter): @@ -1374,7 +1403,7 @@ def test_C7N_CELFilter_shield_protection(celfilter_instance): call('shield', region_name='us-east-1') ] assert mock_filter.get_type_protections.mock_calls == [ - call(celfilter_instance['shield_client'], mock_filter.manager.get_model()) + call(mock_manager['shield_client'], mock_filter.manager.get_model()) ] @@ -1389,7 +1418,7 @@ def test_C7N_CELFilter_shield_subscription(celfilter_instance): ] -def test_C7N_CELFilter_web_acls(celfilter_instance): +def test_C7N_CELFilter_web_acls(celfilter_instance, mock_manager): mock_filter = celfilter_instance['the_filter'] distribution_doc = {"ResourceType": "distribution", "arn": "arn:us-east-1:app-elb:123456789:etc"} with celpy.c7nlib.C7NContext(filter=mock_filter): @@ -1400,6 +1429,6 @@ def test_C7N_CELFilter_web_acls(celfilter_instance): assert mock_filter.manager.get_resource_manager.mock_calls == [ call("waf") ] - assert celfilter_instance['waf_resource_manager'].resources.mock_calls == [ + assert mock_manager['waf_resource_manager'].resources.mock_calls == [ call() ] diff --git a/tests/test_celtypes.py b/tests/test_celtypes.py index f34b587..f8cdd68 100644 --- a/tests/test_celtypes.py +++ b/tests/test_celtypes.py @@ -19,7 +19,7 @@ import math from unittest.mock import sentinel -from pytest import * +import pytest from celpy import Int32Value from celpy.celtypes import * @@ -32,7 +32,7 @@ def test_bool_type(): assert logical_condition(t, sentinel.true, sentinel.false) == sentinel.true assert logical_condition(f, sentinel.true, sentinel.false) == sentinel.false - with raises(TypeError): + with pytest.raises(TypeError): logical_condition(StringType("nope"), sentinel.true, sentinel.false) assert logical_and(t, t) == t @@ -43,7 +43,7 @@ def test_bool_type(): assert logical_and(exc, t) == exc assert logical_and(f, exc) == f assert logical_and(exc, f) == f - with raises(TypeError): + with pytest.raises(TypeError): logical_and(exc, StringType("nope")) assert logical_or(t, t) == t @@ -54,17 +54,17 @@ def test_bool_type(): assert logical_or(exc, t) == t assert logical_or(f, exc) == exc assert logical_or(exc, f) == exc - with raises(TypeError): + with pytest.raises(TypeError): logical_or(exc, StringType("nope")) assert logical_not(t) == f assert logical_not(f) == t - with raises(TypeError): + with pytest.raises(TypeError): logical_not(StringType("nope")) assert repr(f) == "BoolType(False)" assert repr(t) == "BoolType(True)" - with raises(TypeError): + with pytest.raises(TypeError): -t assert hash(t) == hash(t) assert hash(t) != hash(f) @@ -76,7 +76,7 @@ def test_bytes_type(): b_0 = BytesType(b'bytes') b_1 = BytesType('bytes') b_2 = BytesType([98, 121, 116, 101, 115]) - with raises(TypeError): + with pytest.raises(TypeError): BytesType(3.14) assert repr(b_0) == "BytesType(b'bytes')" assert BytesType(None) == BytesType(b'') @@ -89,15 +89,15 @@ def test_double_type(): assert repr(d_pi) == "DoubleType(3.1415926)" assert str(d_pi) == "3.1415926" assert -d_pi == -3.1415926 - with raises(TypeError): + with pytest.raises(TypeError): d_pi % d_e - with raises(TypeError): + with pytest.raises(TypeError): 2 % d_e assert d_pi / DoubleType(0.0) == float("inf") assert math.isclose(d_pi / d_e, 3.1415926 / 2.718281828) assert d_pi == d_pi assert d_pi != d_e - with raises(TypeError): + with pytest.raises(TypeError): d_pi == StringType("nope") assert hash(d_pi) == hash(d_pi) assert hash(d_pi) != hash(d_e) @@ -117,9 +117,9 @@ def test_int_type(): assert IntType("-0x2a") == -42 assert IntType("42") == 42 assert IntType("-42") == -42 - with raises(ValueError): + with pytest.raises(ValueError): IntType(9223372036854775807) + IntType(1) - with raises(ValueError): + with pytest.raises(ValueError): -IntType(9223372036854775808) - IntType(1) assert id(i_42) == id(IntType(i_42)) assert repr(i_42) == "IntType(42)" @@ -160,23 +160,23 @@ def test_uint_type(): u_42 = UintType(42) u_max = UintType(18446744073709551615) assert UintType(DoubleType(1.9)) == UintType(2) - with raises(ValueError): + with pytest.raises(ValueError): assert UintType(DoubleType(-123.456)) == UintType(-123) assert UintType(TimestampType("2009-02-13T23:31:30Z")) == 1234567890 assert UintType("0x2a") == 42 - with raises(ValueError): + with pytest.raises(ValueError): assert UintType("-0x2a") == -42 assert UintType("42") == 42 - with raises(ValueError): + with pytest.raises(ValueError): assert UintType("-42") == -42 - with raises(ValueError): + with pytest.raises(ValueError): UintType(18446744073709551615) + UintType(1) - with raises(ValueError): + with pytest.raises(ValueError): UintType(0) - UintType(1) assert id(u_42) == id(UintType(u_42)) assert repr(u_42) == "UintType(42)" assert str(u_max) == "18446744073709551615" - with raises(TypeError): + with pytest.raises(TypeError): assert -UintType("42") assert u_42 == u_42 + UintType(1) - UintType(1) assert u_42 == u_42 * UintType(2) / UintType(2) @@ -207,27 +207,27 @@ def test_list_type(): l_1 = ListType([IntType(42), IntType(6), IntType(7)]) l_2 = ListType([IntType(42), StringType("2.718281828459045**1.791759469228055"), IntType(7)]) assert l_1 == l_1 - with raises(TypeError): + with pytest.raises(TypeError): assert l_1 != l_2 - with raises(TypeError): + with pytest.raises(TypeError): assert not l_1 == l_2 assert repr(l_1) == "ListType([IntType(42), IntType(6), IntType(7)])" - with raises(TypeError): + with pytest.raises(TypeError): l_1 < l_2 - with raises(TypeError): + with pytest.raises(TypeError): l_1 <= l_2 - with raises(TypeError): + with pytest.raises(TypeError): l_1 > l_2 - with raises(TypeError): + with pytest.raises(TypeError): l_1 >= l_2 - with raises(TypeError): + with pytest.raises(TypeError): l_1 == DoubleType("42.0") - with raises(TypeError): + with pytest.raises(TypeError): assert l_1 != DoubleType("42.0") assert l_1 != ListType([IntType(42), IntType(42), IntType(42)]) assert ListType() == ListType([]) assert ListType(ListType([IntType(42)])) == ListType([IntType(42)]) - + assert l_1.contains(IntType(42)) def test_map_type(): m_0 = MapType() @@ -251,36 +251,45 @@ def test_map_type(): assert m_1 == m_3 assert not m_1 != m_1 assert not m_single != m_single - with raises(TypeError): + with pytest.raises(TypeError): MapType(3.1415926) - with raises(TypeError): + with pytest.raises(TypeError): assert m_1 != m_2 - with raises(TypeError): + with pytest.raises(TypeError): assert not m_1 == m_2 assert repr(m_1) == ( "MapType({StringType('A'): IntType(42), " "StringType('X'): IntType(6), " "StringType('Y'): IntType(7)})") assert m_1[StringType("A")] == IntType(42) - with raises(TypeError): + with pytest.raises(TypeError): m_1[ListType([StringType("A")])] - with raises(TypeError): + with pytest.raises(TypeError): + m_1.get(ListType([StringType("A")])) + with pytest.raises(TypeError): m_1 < m_2 - with raises(TypeError): + with pytest.raises(TypeError): m_1 <= m_2 - with raises(TypeError): + with pytest.raises(TypeError): m_1 > m_2 - with raises(TypeError): + with pytest.raises(TypeError): m_1 >= m_2 - with raises(TypeError): + with pytest.raises(TypeError): m_1 == DoubleType("42.0") - with raises(TypeError): + with pytest.raises(TypeError): assert m_1 != DoubleType("42.0") assert m_1 != MapType({ StringType("A"): IntType(42), StringType("X"): IntType(42), StringType("Y"): IntType(42)} ) + assert m_1.contains(StringType("A")) + assert m_1.get(StringType("A")) == IntType(42) + assert m_1.get(StringType("NOT_FOUND"), IntType(21)) == IntType(21) + with pytest.raises(KeyError): + m_1.get(StringType("NOT_FOUND")) + with pytest.raises(ValueError): + m_bad = MapType([("A", IntType(42)), ("A", StringType("Duplicate"))]) def test_string_type(): @@ -294,6 +303,7 @@ def test_string_type(): assert s_1 != s_2 assert id(s_1) == id(s_1) assert id(s_1) != id(s_2) + assert s_2.contains("str") def test_string_issue_48(): @@ -311,29 +321,29 @@ def test_timestamp_type(): assert ts_1 == ts_1_dt assert ts_1 == ts_1_tuple assert ts_1 == ts_1_m - with raises(ValueError): + with pytest.raises(ValueError): TimestampType("2009-02-13T23:31:xyZ") - with raises(TypeError): + with pytest.raises(TypeError): TimestampType(IntType(42)) assert repr(ts_1) == repr(ts_1_m) assert str(ts_1) == "2009-02-13T23:31:30Z" assert TimestampType(2009, 2, 13, 23, 31, 0) + DurationType("30s") == ts_1 - with raises(TypeError): + with pytest.raises(TypeError): assert TimestampType(2009, 2, 13, 23, 31, 0) + StringType("30s") assert DurationType("30s") + TimestampType(2009, 2, 13, 23, 31, 0) == ts_1 - with raises(TypeError): + with pytest.raises(TypeError): assert StringType("30s") + TimestampType(2009, 2, 13, 23, 31, 0) assert ( TimestampType(2009, 2, 13, 0, 0, 0) - TimestampType(2009, 1, 1, 0, 0, 0) == DurationType(datetime.timedelta(days=43)) ) - with raises(TypeError): + with pytest.raises(TypeError): assert TimestampType(2009, 2, 13, 23, 31, 0) - StringType("30s") assert TimestampType(2009, 2, 13, 23, 32, 0) - DurationType("30s") == ts_1 assert ts_1.getDate() == IntType(13) assert ts_1.getDate("+00:00") == IntType(13) - with raises(ValueError): + with pytest.raises(ValueError): assert ts_1.getDate("+no:pe") == IntType(13) assert ts_1.getDayOfMonth() == IntType(12) assert ts_1.getDayOfWeek() == IntType(5) @@ -375,15 +385,15 @@ def test_duration_type(): d_1 = DurationType("43200s") assert d_1 == d_1_dt assert d_1 == d_1_tuple - with raises(ValueError): + with pytest.raises(ValueError): DurationType(datetime.timedelta(seconds=315576000001)) - with raises(ValueError): + with pytest.raises(ValueError): DurationType("not:a:duration") - with raises(ValueError): + with pytest.raises(ValueError): DurationType("315576000001s") - with raises(ValueError): + with pytest.raises(ValueError): DurationType(IntType(315576000001)) - with raises(TypeError): + with pytest.raises(TypeError): DurationType({"Some": "JSON"}) assert repr(d_1) == "DurationType('43200s')" assert str(d_1) == "43200s" @@ -396,13 +406,13 @@ def test_duration_type(): # See https://github.com/google/cel-spec/issues/138 assert DurationType("+2m30s").getSeconds() == IntType(150) assert DurationType("-2m30s").getSeconds() == IntType(-150) - with raises(ValueError): + with pytest.raises(ValueError): DurationType("-2w30z") def test_function_type(): f_1 = FunctionType() - with raises(NotImplementedError): + with pytest.raises(NotImplementedError): f_1(IntType(0)) @@ -415,14 +425,13 @@ def test_int32_value(): def test_type_type(): - t_1 = TypeType("DOUBLE") t_2 = TypeType("IntType") - with raises(TypeError): - t_3 = TypeType("not_a_type") + t_3 = TypeType("not_a_type") t_4 = TypeType(DoubleType(3.14159)) - assert t_1 == t_1 - assert t_1 != t_2 - assert t_1 == t_4 + assert t_2 == t_2 + assert t_2 == t_3 + assert t_2 != t_4 + assert TypeType(DoubleType) is TypeType def test_message_type(): @@ -430,5 +439,5 @@ def test_message_type(): assert mt_1["name"] == IntType(42) mt_2 = MessageType(value=IntType(42)) assert mt_2["value"] == IntType(42) - with raises(TypeError): + with pytest.raises(TypeError): MessageType("not", "good") diff --git a/tests/test_evaluation.py b/tests/test_evaluation.py index 6b0927d..f302722 100644 --- a/tests/test_evaluation.py +++ b/tests/test_evaluation.py @@ -15,11 +15,11 @@ """ Test all the evaluation methods. -A large number of tests use :py:meth:`Evaluator.evaluate`. +A large number of tests exercise :py:meth:`Evaluator.evaluate`. -This may not be ideal from a unit testing perspective. This approach -tends to test the superclass :py:meth:`lark.visitors.Interpreter.visit`, -which involves testing a fair amount of Lark code. +The approach used here may not be ideal from a unit testing perspective. +This approach tends to test the superclass :py:meth:`lark.visitors.Interpreter.visit`, +which involves re-testing a fair amount of Lark code. The alternative is to test methods in detail. For example, using :py:meth:`Evaluator.expr` directly with a :py:class:`lark.Tree` object and @@ -114,18 +114,20 @@ def mock_operation(a, b): elif a == sentinel.OtherError: raise ValueError(sentinel.value_error_message) else: - return sentinel.c + return sentinel.A_OP_B - result_1 = mock_operation(sentinel.TypeError, sentinel.b) + result_1 = mock_operation(sentinel.TypeError, sentinel.VALUE) + assert isinstance(result_1, CELEvalError) assert result_1.args == (sentinel.eval_message, TypeError, (sentinel.type_error_message,)) assert result_1.__cause__.__class__ == TypeError with pytest.raises(ValueError) as exc_info: - result_2 = mock_operation(sentinel.OtherError, sentinel.b) + result_2 = mock_operation(sentinel.OtherError, sentinel.VALUE) + assert not isinstance(exc_info.value, CELEvalError) assert exc_info.value.args == (sentinel.value_error_message,) - result_3 = mock_operation(sentinel.a, sentinel.b) - assert result_3 == sentinel.c + result_3 = mock_operation(sentinel.A, sentinel.B) + assert result_3 == sentinel.A_OP_B def test_boolean_decorator(): @@ -146,6 +148,10 @@ def mock_operation(a, b): def test_operator_in(): + """ + Was a separate function. + Revised with 0.4.0 release to be method of String,Type ListType, and MapType. + """ container_1 = celtypes.ListType([ celtypes.IntType(42), celtypes.IntType(6), @@ -207,6 +213,7 @@ def test_referent(): assert r_1.annotation is celtypes.IntType assert r_1.container is None assert r_1.value is celtypes.IntType + assert r_0 != r_1 r_1.value = celtypes.IntType(42) assert r_1.annotation is celtypes.IntType assert r_1.container is None @@ -218,6 +225,7 @@ def test_referent(): assert r_1.container is nc assert r_1.value == nc # preferred over the assigned value. r_c = r_1.clone() + assert r_c == r_1 assert r_c.container == r_1.container assert r_c.value == r_1.value @@ -232,8 +240,15 @@ def test_name_container(): "a.b.c": celtypes.StringType("yeah"), } ) - member_dot = nc.resolve_name("", "a").resolve_name("", "b").resolve_name("", "c") - assert member_dot == celtypes.StringType("yeah") + assert nc.find_name([]).value == nc + assert nc.find_name(["a","b","c"]).value == celtypes.StringType("yeah") + member_dot = nc.resolve_name("", "a").container.resolve_name("", "b").container.resolve_name("", "c") + assert member_dot.value == celtypes.StringType("yeah") + + nc2 = NameContainer() + nc2.load_values({"answer": celtypes.IntType(42)}) + with pytest.raises(TypeError): + nc2.find_name(["answer", "nope"]).value def test_name_container_init(): @@ -260,6 +275,13 @@ def test_activation_no_package_no_vars(): assert a_n.resolve_variable("x") == celtypes.IntType(42) +def test_activation_annotation_no_vars(): + a = Activation(annotations={"answer": celtypes.IntType}) + with pytest.raises(KeyError) as exc_info: + print(f"Found {a.undefined=}") + assert exc_info.value.args == ("undefined",) + + def test_activation_jq_package_vars(): """ This test pushes the envelope a on what CEL should be able to do. @@ -340,7 +362,6 @@ def test_activation_bad_dot_name_syntax(): } ) - @pytest.fixture def mock_tree(): tree = Mock( @@ -352,10 +373,6 @@ def mock_tree(): ) return tree -def test_find_ident(mock_tree): - fi = FindIdent.in_tree(mock_tree) - assert fi == sentinel.ident - def test_trace_decorator(mock_tree): @@ -376,19 +393,6 @@ def method(self, tree): call('%s%s -> %r', '| ', 'ident', sentinel.result) ] -def test_evaluator_init(): - tree = lark.Tree(data="literal", children=[]) - activation = Mock() - def override(a): - return celtypes.BoolType(False) - e_0 = Evaluator(tree, activation) - assert e_0.functions == celpy.evaluation.base_functions - e_1 = Evaluator(tree, activation, functions=[override]) - assert e_1.functions['override'] == override - e_2 = Evaluator(tree, activation, functions={"override": override}) - assert e_2.functions['override'] == override - - def test_set_activation(): tree = lark.Tree(data="literal", children=[]) activation = Activation( @@ -409,55 +413,135 @@ def test_set_activation(): assert e_0.ident_value('int') == celtypes.IntType -def test_function_eval(monkeypatch): +def test_function_eval_0(monkeypatch): tree = lark.Tree( data="primary", children=[] ) - activation = Mock() + activation = Mock(spec=Activation, resolve_function=Mock(side_effect=KeyError)) evaluator = Evaluator( tree, activation ) - assert isinstance(evaluator.function_eval(lark.Token("IDENT", "nope")), CELEvalError) + + unknown_function_result = evaluator.function_eval(lark.Token("IDENT", "nope")) + print(f"{unknown_function_result=}") + assert isinstance(unknown_function_result, CELEvalError) + +def test_function_eval_1(monkeypatch): + tree = lark.Tree( + data="primary", + children=[] + ) + activation = Mock(spec=Activation, resolve_function=Mock(return_value=function_size)) + evaluator = Evaluator( + tree, + activation + ) + error = CELEvalError(sentinel.message) - assert evaluator.function_eval(lark.Token("IDENT", "size"), error) == error + function_of_error_result = evaluator.function_eval(lark.Token("IDENT", "size"), error) + print(f"{function_of_error_result=}") + assert function_of_error_result == error + +def test_function_eval_2(monkeypatch): + mock_size = Mock(side_effect=ValueError(sentinel.value)) + tree = lark.Tree( + data="primary", + children=[] + ) + activation = Mock(spec=Activation, + resolve_function=Mock(return_value=mock_size)) + evaluator = Evaluator( + tree, + activation + ) - monkeypatch.setitem(evaluator.functions, "size", Mock(side_effect=ValueError(sentinel.value))) value = evaluator.function_eval(lark.Token("IDENT", "size")) assert isinstance(value, CELEvalError) assert value.args[1] == ValueError, f"{value.args} != (..., ValueError, ...)" assert value.args[2] == (sentinel.value,), f"{value.args} != (..., ..., (sentinel.value,))" - monkeypatch.setitem(evaluator.functions, "size", Mock(side_effect=TypeError(sentinel.type))) +def test_function_eval_3(monkeypatch): + mock_size = Mock(side_effect=TypeError(sentinel.type)) + tree = lark.Tree( + data="primary", + children=[] + ) + activation = Mock(spec=Activation, + resolve_function=Mock(return_value=mock_size)) + evaluator = Evaluator( + tree, + activation + ) + value = evaluator.function_eval(lark.Token("IDENT", "size")) assert isinstance(value, CELEvalError) assert value.args[1] == TypeError, f"{value.args} != (..., TypeError, ...)" assert value.args[2] == (sentinel.type,), f"{value.args} != (..., ..., (sentinel.type,))" -def test_method_eval(monkeypatch): +def test_method_eval_0(monkeypatch): tree = lark.Tree( data="primary", children=[] ) - activation = Mock() + activation = Mock(spec=Activation, functions=sentinel.FUNCTIONS, resolve_function=Mock(side_effect=KeyError)) evaluator = Evaluator( tree, activation ) - assert isinstance(evaluator.method_eval(None, lark.Token("IDENT", "nope")), CELEvalError) + + unknown_method_result = evaluator.method_eval(None, lark.Token("IDENT", "nope")) + print(f"{unknown_method_result=}") + assert isinstance(unknown_method_result, CELEvalError) + +def test_method_eval_1(monkeypatch): + tree = lark.Tree( + data="primary", + children=[] + ) + activation = Mock(spec=Activation, resolve_function=Mock(return_value=function_size)) + evaluator = Evaluator( + tree, + activation + ) + error = CELEvalError(sentinel.message) assert evaluator.method_eval(error, lark.Token("IDENT", "size"), None) == error assert evaluator.method_eval(None, lark.Token("IDENT", "size"), error) == error - monkeypatch.setitem(evaluator.functions, "size", Mock(side_effect=ValueError(sentinel.value))) +def test_method_eval_2(monkeypatch): + mock_function = Mock(side_effect=ValueError(sentinel.value)) + tree = lark.Tree( + data="primary", + children=[] + ) + activation = Mock(spec=Activation, + resolve_function=Mock(return_value=mock_function)) + evaluator = Evaluator( + tree, + activation + ) + value = evaluator.method_eval(None, lark.Token("IDENT", "size")) assert isinstance(value, CELEvalError) assert value.args[1] == ValueError, f"{value.args} != (..., ValueError, ...)" assert value.args[2] == (sentinel.value,), f"{value.args} != (..., ..., (sentinel.value,))" - monkeypatch.setitem(evaluator.functions, "size", Mock(side_effect=TypeError(sentinel.type))) +def test_method_eval_3(monkeypatch): + mock_function = Mock(side_effect=TypeError(sentinel.type)) + tree = lark.Tree( + data="primary", + children=[] + ) + activation = Mock(spec=Activation, + resolve_function=Mock(return_value=mock_function)) + evaluator = Evaluator( + tree, + activation + ) + value = evaluator.method_eval(None, lark.Token("IDENT", "size")) assert isinstance(value, CELEvalError) assert value.args[1] == TypeError, f"{value.args} != (..., TypeError, ...)" @@ -536,7 +620,7 @@ def mock_left_expr_tree(): def test_eval_expr_3_left_good(mock_left_expr_tree): """Assert ``true ? 6 : invalid`` does not execute the invalid expression.""" - activation = Mock() + activation = Mock(resolve_function=Mock(return_value=celpy.celtypes.logical_condition)) evaluator = Evaluator( mock_left_expr_tree, activation @@ -548,11 +632,10 @@ def test_eval_expr_3_left_good(mock_left_expr_tree): def test_eval_expr_3_bad_override(mock_left_expr_tree): def bad_condition(a, b, c): raise TypeError - activation = Mock() + activation = Mock(resolve_function=Mock(return_value=bad_condition)) evaluator = Evaluator( mock_left_expr_tree, activation, - functions={"_?_:_": bad_condition} ) with pytest.raises(celpy.evaluation.CELEvalError) as exc_info: evaluator.evaluate() @@ -561,11 +644,10 @@ def bad_condition(a, b, c): def test_eval_expr_3_bad_cond_value(mock_left_expr_tree): def bad_condition(a, b, c): raise celpy.evaluation.CELEvalError("Baseline Error") - activation = Mock() + activation = Mock(resolve_function=Mock(return_value=bad_condition)) evaluator = Evaluator( mock_left_expr_tree, activation, - functions={"_?_:_": bad_condition} ) with pytest.raises(celpy.evaluation.CELEvalError) as exc_info: evaluator.evaluate() @@ -598,7 +680,7 @@ def mock_right_expr_tree(): def test_eval_expr_3_right_good(mock_right_expr_tree): """Assert ``false ? invalid : 7`` does not execute the invalid expression.""" - activation = Mock() + activation = Mock(resolve_function=Mock(return_value=celtypes.logical_condition)) evaluator = Evaluator( mock_right_expr_tree, activation @@ -670,7 +752,7 @@ def binop_2_tree(data, lit_type, lit_value_1, lit_value_2): def test_eval_conditionalor_2_good(): tree = binop_2_tree("conditionalor", "BOOL_LIT", "false", "true") - activation = Mock() + activation = Mock(resolve_function=Mock(return_value=celtypes.logical_or)) evaluator = Evaluator( tree, activation @@ -682,11 +764,11 @@ def test_eval_conditionalor_2_bad_override(): def bad_logical_or(a, b): raise TypeError tree = binop_2_tree("conditionalor", "BOOL_LIT", "false", "true") - activation = Mock() + activation = Mock(resolve_function=Mock(return_value=bad_logical_or)) evaluator = Evaluator( tree, activation, - functions={"_||_": bad_logical_or} + # functions={"_||_": bad_logical_or} ) with pytest.raises(celpy.evaluation.CELEvalError): evaluator.evaluate() @@ -722,7 +804,7 @@ def test_eval_conditionaland_1(): def test_eval_conditionaland_2_good(): tree = binop_2_tree("conditionaland", "BOOL_LIT", "false", "true") - activation = Mock() + activation = Mock(resolve_function=Mock(return_value=celtypes.logical_and)) evaluator = Evaluator( tree, activation @@ -734,11 +816,10 @@ def test_eval_conditionaland_2_bad_override(): def bad_logical_and(a, b): raise TypeError tree = binop_2_tree("conditionaland", "BOOL_LIT", "false", "true") - activation = Mock() + activation = Mock(resolve_function=Mock(return_value=bad_logical_and)) evaluator = Evaluator( tree, activation, - functions={"_&&_": bad_logical_and} ) with pytest.raises(celpy.evaluation.CELEvalError): evaluator.evaluate() @@ -859,8 +940,8 @@ def test_binops(binop_trees): to provide an override for. """ t_0, t_1, t_2, expected, function = binop_trees - activation = Mock() + activation = Mock(resolve_function=Mock(return_value=base_functions[function])) evaluator_0 = Evaluator( t_0, activation @@ -886,10 +967,10 @@ def test_binops(binop_trees): def bad_function(a, b): raise TypeError + activation = Mock(resolve_function=Mock(return_value=bad_function)) evaluator_2_b = Evaluator( t_2, activation, - functions={function: bad_function} ) with pytest.raises(celpy.evaluation.CELEvalError): evaluator_2_b.evaluate() @@ -965,7 +1046,7 @@ def test_unops(unop_trees): to provide an override for. """ t_0, t_1, t_2, expected, function = unop_trees - activation = Mock() + activation = Mock(resolve_function=Mock(return_value=base_functions[function])) evaluator_0 = Evaluator( t_0, @@ -992,19 +1073,20 @@ def test_unops(unop_trees): def bad_function(a, b): raise TypeError + activation = Mock(resolve_function=Mock(return_value=bad_function)) evaluator_2_b = Evaluator( t_2, activation, - functions={function: bad_function} ) with pytest.raises(celpy.evaluation.CELEvalError): evaluator_2_b.evaluate() -# The following use a patch to :py:meth:`Evaluator.visit_children` to produce useful answers. -# It simplifies the required :py:class:`lark.Tree` object. - def test_member(monkeypatch): + """ + Uses a patch to :py:meth:`Evaluator.visit_children` to produce useful answers. + It simplifies the required :py:class:`lark.Tree` object. + """ visit_children = Mock(return_value=[celtypes.IntType(42)]) monkeypatch.setattr(Evaluator, 'visit_children', visit_children) tree = lark.Tree( @@ -1231,7 +1313,7 @@ def test_member_dot_package(monkeypatch): def test_member_dot_missing_package(monkeypatch): - """Activation has ``{"name1": {"name2": Annotation}}`` created from "name1.name2". + """Activation has ``{"name1": {"not the expected name": Annotation}}`` created from "name1.not the expected name". To get a parse tree:: @@ -1273,7 +1355,7 @@ def test_member_dot_missing_package(monkeypatch): def test_member_dot_arg_method_0(monkeypatch): - """A method, e.g., Timestamp(1234567890).getMonth(); distinct from the macros.""" + """A method, e.g., timestamp("2009-02-13T23:31:30Z").getMonth(); distinct from the macros.""" visit_children = Mock( return_value=[ celtypes.TimestampType("2009-02-13T23:31:30Z"), @@ -1300,7 +1382,7 @@ def test_member_dot_arg_method_0(monkeypatch): ) evaluator_0 = Evaluator( tree, - activation=Mock() + activation=Mock(resolve_function=Mock(return_value=function_getMonth)) ) assert evaluator_0.member_dot_arg(tree.children[0]) == celtypes.IntType(1) @@ -1337,7 +1419,7 @@ def test_member_dot_arg_method_1(monkeypatch): ) evaluator_0 = Evaluator( tree, - activation=Mock() + activation=Mock(resolve_function=Mock(return_value=function_contains)) ) assert evaluator_0.member_dot_arg(tree.children[0]) == celtypes.BoolType(True) @@ -1350,18 +1432,16 @@ def test_build_macro_eval(monkeypatch): mock_evaluator_class = Mock( return_value=Mock( + # OLD DESIGN set_activation=Mock( return_value=Mock( evaluate=Mock(return_value=sentinel.output) ) - ) + ), + evaluate=Mock(return_value=sentinel.output) ) ) monkeypatch.setattr(celpy.evaluation, 'Evaluator', mock_evaluator_class) - mock_find_ident_class = Mock( - in_tree=Mock(return_value=sentinel.variable) - ) - monkeypatch.setattr(celpy.evaluation, 'FindIdent', mock_find_ident_class) sub_expression = lark.Tree(data="expr", children=[]) child = lark.Tree( @@ -1383,27 +1463,17 @@ def test_build_macro_eval(monkeypatch): ) subexpr = evaluator_0.build_macro_eval(child) - # `FindIdent` walked the tree to locate the variable name. - assert mock_find_ident_class.in_tree.mock_calls == [ - call(lark.Tree("ident", [lark.Token("IDENT", 'variable')])) - ] - # Nested `Evaluator` instance created assert mock_evaluator_class.mock_calls == [ - call(sub_expression, activation=evaluator_0.activation, functions=evaluator_0.functions) + call(sub_expression, activation=evaluator_0.activation) # , functions=evaluator_0.functions) ] # When we evaluated the sub-expression created, it uses the nest `Evaluator` instance. assert subexpr(sentinel.input) == sentinel.output # The nested evaluator's top-level activation had the input value set. - assert mock_evaluator_class.return_value.set_activation.mock_calls == [ - call({sentinel.variable: sentinel.input}) - ] - - # And. The nested evaluator's evaluate() was used to produce the answer. - assert mock_evaluator_class.return_value.set_activation.return_value.evaluate.mock_calls == [ - call() + assert mock_evaluator_class.return_value.evaluate.mock_calls == [ + call({'variable': sentinel.input}) ] @@ -1415,18 +1485,16 @@ def test_build_ss_macro_eval(monkeypatch): mock_evaluator_class = Mock( return_value=Mock( + # OLD DESIGN set_activation=Mock( return_value=Mock( evaluate=Mock(side_effect=[sentinel.output, CELEvalError]) ) - ) + ), + evaluate=Mock(side_effect=[sentinel.output, CELEvalError]) ) ) monkeypatch.setattr(celpy.evaluation, 'Evaluator', mock_evaluator_class) - mock_find_ident_class = Mock( - in_tree=Mock(return_value=sentinel.variable) - ) - monkeypatch.setattr(celpy.evaluation, 'FindIdent', mock_find_ident_class) sub_expression = lark.Tree(data="expr", children=[]) child = lark.Tree( @@ -1448,14 +1516,9 @@ def test_build_ss_macro_eval(monkeypatch): ) subexpr = evaluator_0.build_ss_macro_eval(child) - # `FindIdent` walked the tree to locate the variable name. - assert mock_find_ident_class.in_tree.mock_calls == [ - call(lark.Tree("ident", [lark.Token("IDENT", 'variable')])) - ] - # Nested `Evaluator` instance created assert mock_evaluator_class.mock_calls == [ - call(sub_expression, activation=evaluator_0.activation, functions=evaluator_0.functions) + call(sub_expression, activation=evaluator_0.activation) # , functions=evaluator_0.functions) ] # When we evaluated the sub-expression created, it uses the nest `Evaluator` instance. @@ -1467,15 +1530,9 @@ def test_build_ss_macro_eval(monkeypatch): assert isinstance(subexpr(sentinel.input), CELEvalError) # The nested evaluator's top-level activation had the input value set. - assert mock_evaluator_class.return_value.set_activation.mock_calls == [ - call({sentinel.variable: sentinel.input}), - call({sentinel.variable: sentinel.input}), - ] - - # And. The nested evaluator's evaluate() was used to produce the answer. - assert mock_evaluator_class.return_value.set_activation.return_value.evaluate.mock_calls == [ - call(), - call(), + assert mock_evaluator_class.return_value.evaluate.mock_calls == [ + call({'variable': sentinel.input}), + call({'variable': sentinel.input}), ] def test_build_reduce_macro_eval(monkeypatch): @@ -1486,18 +1543,16 @@ def test_build_reduce_macro_eval(monkeypatch): mock_evaluator_class = Mock( return_value=Mock( + # OLD DESIGN set_activation=Mock( return_value=Mock( evaluate=Mock(return_value=sentinel.output) ) - ) + ), + evaluate=Mock(return_value=sentinel.output) ) ) monkeypatch.setattr(celpy.evaluation, 'Evaluator', mock_evaluator_class) - mock_find_ident_class = Mock( - in_tree=Mock(return_value=sentinel.variable) - ) - monkeypatch.setattr(celpy.evaluation, 'FindIdent', mock_find_ident_class) sub_expression_1 = lark.Tree(data="expr", children=["1"]) sub_expression_2 = lark.Tree(data="expr", children=["2"]) @@ -1522,15 +1577,9 @@ def test_build_reduce_macro_eval(monkeypatch): ) subexpr, init_value = evaluator_0.build_reduce_macro_eval(child) - # `FindIdent` walked the tree to locate the variable name. - assert mock_find_ident_class.in_tree.mock_calls == [ - call(lark.Tree("ident", [lark.Token("IDENT", 'r')])), - call(lark.Tree("ident", [lark.Token("IDENT", 'i')])) - ] - # Nested `Evaluator` instance created assert mock_evaluator_class.mock_calls == [ - call(sub_expression_2, activation=evaluator_0.activation, functions=evaluator_0.functions) + call(sub_expression_2, activation=evaluator_0.activation) # , functions=evaluator_0.functions) ] # init_value is the sub_expression @@ -1540,17 +1589,13 @@ def test_build_reduce_macro_eval(monkeypatch): assert subexpr(sentinel.input1, sentinel.input2) == sentinel.output # The nested evaluator's top-level activation had the input value set. - assert mock_evaluator_class.return_value.set_activation.mock_calls == [ - call({sentinel.variable: sentinel.input2}) - ] - - # And. The nested evaluator's evaluate() was used to produce the answer. - assert mock_evaluator_class.return_value.set_activation.return_value.evaluate.mock_calls == [ - call() + assert mock_evaluator_class.return_value.evaluate.mock_calls == [ + call({'r': sentinel.input1, 'i': sentinel.input2}) ] def macro_member_tree(macro_name, *args): + """Creates 'placeholder.macro_name(*args)' default args are variable, 0.""" tree = lark.Tree( data="member", children=[ @@ -1835,7 +1880,7 @@ def test_member_index(index_trees): evaluator_0 = Evaluator( tree, - activation=Mock() + activation=Mock(resolve_function=Mock(return_value=operator.getitem)) ) if isinstance(expected, type): # Does the member_index() method produce a CELEvalError instance? @@ -2267,10 +2312,11 @@ def test_primary_ident_arg_empty(monkeypatch): ], meta=Mock(line=1, column=1) ) + shake_hands_function = Mock(return_value=sentinel.value) evaluator_0 = Evaluator( tree, - activation=Mock(), - functions = {"shake_hands": Mock(return_value=sentinel.value)} + activation=Mock(resolve_function=Mock(return_value=shake_hands_function)), + # functions = {"shake_hands": Mock(return_value=sentinel.value)} ) assert evaluator_0.primary(tree) == sentinel.value assert visit_children.mock_calls == [call(lark.Tree("exprlist", []))] diff --git a/tests/test_main.py b/tests/test_main.py index e111d53..4cb28d9 100644 --- a/tests/test_main.py +++ b/tests/test_main.py @@ -25,18 +25,21 @@ """ import argparse +import datetime import io +import stat as os_stat +from pathlib import Path import sys -from unittest.mock import Mock, call, sentinel +from unittest.mock import Mock, call, sentinel, ANY -from pytest import * +import pytest import celpy import celpy.__main__ from celpy import celtypes -@fixture +@pytest.fixture def mock_os_environ(monkeypatch): monkeypatch.setitem(celpy.__main__.os.environ, "OS_ENV_VAR", "3.14") @@ -58,7 +61,7 @@ def test_arg_type_value(mock_os_environ): celtypes.DoubleType, 3.14, ) - with raises(argparse.ArgumentTypeError): + with pytest.raises(argparse.ArgumentTypeError): celpy.__main__.arg_type_value("name:type:value") @@ -84,7 +87,7 @@ def test_get_options(): def test_arg_type_bad(capsys, monkeypatch): """GIVEN invalid arg values; WHEN parsing; THEN correct interpretation.""" monkeypatch.setenv("COLUMNS", "80") - with raises(SystemExit) as exc_info: + with pytest.raises(SystemExit) as exc_info: options = celpy.__main__.get_options( ["--arg", "name:nope=42", "-n", "355./113."] ) @@ -105,7 +108,7 @@ def test_arg_type_bad(capsys, monkeypatch): def test_arg_value_bad(capsys, monkeypatch): """GIVEN invalid arg values; WHEN parsing; THEN correct interpretation.""" monkeypatch.setenv("COLUMNS", "80") - with raises(SystemExit) as exc_info: + with pytest.raises(SystemExit) as exc_info: options = celpy.__main__.get_options( ["--arg", "name:int=nope", "-n", "355./113."] ) @@ -128,7 +131,7 @@ def test_arg_combo_bad(capsys, monkeypatch): " [--json-document NAME] [-b] [-f FORMAT]", " [expr]", ] - with raises(SystemExit) as exc_info: + with pytest.raises(SystemExit) as exc_info: options = celpy.__main__.get_options( ["-i", "-n", "355./113."] ) @@ -138,7 +141,7 @@ def test_arg_combo_bad(capsys, monkeypatch): "celpy: error: Interactive mode and an expression provided", ] - with raises(SystemExit) as exc_info: + with pytest.raises(SystemExit) as exc_info: options = celpy.__main__.get_options( ["-n"] ) @@ -148,7 +151,7 @@ def test_arg_combo_bad(capsys, monkeypatch): "celpy: error: No expression provided", ] - with raises(SystemExit) as exc_info: + with pytest.raises(SystemExit) as exc_info: options = celpy.__main__.get_options( ["-n", "--json-document=_", "--json-package=_"] ) @@ -159,7 +162,7 @@ def test_arg_combo_bad(capsys, monkeypatch): ] -@fixture +@pytest.fixture def mock_cel_environment(monkeypatch): mock_runner = Mock(evaluate=Mock(return_value=str(sentinel.OUTPUT))) mock_env = Mock( @@ -175,10 +178,12 @@ def test_main_0(mock_cel_environment, caplog, capsys): argv = ["--null-input", '"Hello world! I\'m " + name + "."'] status = celpy.__main__.main(argv) assert status == 0 - assert mock_cel_environment.mock_calls == [call(package=None, annotations=None)] + assert mock_cel_environment.mock_calls == [ + call(package=None, annotations={"stat": celpy.celtypes.FunctionType}) + ] env = mock_cel_environment.return_value assert env.compile.mock_calls == [call('"Hello world! I\'m " + name + "."')] - assert env.program.mock_calls == [call(sentinel.AST)] + assert env.program.mock_calls == [call(sentinel.AST, functions={"stat": ANY})] prgm = env.program.return_value assert prgm.evaluate.mock_calls == [call({})] assert caplog.messages == [] @@ -198,11 +203,11 @@ def test_main_1(mock_cel_environment, caplog, capsys): status = celpy.__main__.main(argv) assert status == 0 assert mock_cel_environment.mock_calls == [ - call(package=None, annotations={"name": celtypes.StringType}) + call(package=None, annotations={"name": celtypes.StringType, "stat": celpy.celtypes.FunctionType}) ] env = mock_cel_environment.return_value assert env.compile.mock_calls == [call('"Hello world! I\'m " + name + "."')] - assert env.program.mock_calls == [call(sentinel.AST)] + assert env.program.mock_calls == [call(sentinel.AST, functions={"stat": ANY})] prgm = env.program.return_value assert prgm.evaluate.mock_calls == [call({"name": "CEL"})] assert caplog.messages == [] @@ -218,10 +223,12 @@ def test_main_pipe(mock_cel_environment, caplog, capsys): status = celpy.__main__.main(argv) sys.stdin = sys.__stdin__ assert status == 0 - assert mock_cel_environment.mock_calls == [call(package="jq", annotations=None)] + assert mock_cel_environment.mock_calls == [ + call(package="jq", annotations={"stat": celpy.celtypes.FunctionType}) + ] env = mock_cel_environment.return_value assert env.compile.mock_calls == [call('"Hello world! I\'m " + name + "."')] - assert env.program.mock_calls == [call(sentinel.AST)] + assert env.program.mock_calls == [call(sentinel.AST, functions={'stat': ANY})] prgm = env.program.return_value assert prgm.evaluate.mock_calls == [ call( @@ -246,10 +253,12 @@ def test_main_0_non_boolean(mock_cel_environment, caplog, capsys): argv = ["-bn", '"Hello world! I\'m " + name + "."'] status = celpy.__main__.main(argv) assert status == 2 - assert mock_cel_environment.mock_calls == [call(package=None, annotations=None)] + assert mock_cel_environment.mock_calls == [ + call(package=None, annotations={"stat": celpy.celtypes.FunctionType}) + ] env = mock_cel_environment.return_value assert env.compile.mock_calls == [call('"Hello world! I\'m " + name + "."')] - assert env.program.mock_calls == [call(sentinel.AST)] + assert env.program.mock_calls == [call(sentinel.AST, functions={'stat': ANY})] prgm = env.program.return_value assert prgm.evaluate.mock_calls == [call({})] assert caplog.messages == [ @@ -260,7 +269,7 @@ def test_main_0_non_boolean(mock_cel_environment, caplog, capsys): assert err == "" -@fixture +@pytest.fixture def mock_cel_environment_false(monkeypatch): mock_runner = Mock(evaluate=Mock(return_value=celtypes.BoolType(False))) mock_env = Mock( @@ -281,11 +290,11 @@ def test_main_0_boolean(mock_cel_environment_false, caplog, capsys): status = celpy.__main__.main(argv) assert status == 1 assert mock_cel_environment_false.mock_calls == [ - call(package=None, annotations=None) + call(package=None, annotations={"stat": celpy.celtypes.FunctionType}) ] env = mock_cel_environment_false.return_value assert env.compile.mock_calls == [call("2 == 1")] - assert env.program.mock_calls == [call(sentinel.AST)] + assert env.program.mock_calls == [call(sentinel.AST, functions={'stat': ANY})] prgm = env.program.return_value assert prgm.evaluate.mock_calls == [call({})] assert caplog.messages == [] @@ -294,7 +303,7 @@ def test_main_0_boolean(mock_cel_environment_false, caplog, capsys): assert err == "" -@fixture +@pytest.fixture def mock_cel_environment_integer(monkeypatch): mock_runner = Mock(evaluate=Mock(return_value=celtypes.IntType(3735928559))) mock_env = Mock( @@ -317,11 +326,11 @@ def test_main_slurp_int_format(mock_cel_environment_integer, caplog, capsys): sys.stdin = sys.__stdin__ assert status == 0 assert mock_cel_environment_integer.mock_calls == [ - call(package='jq', annotations=None) + call(package='jq', annotations={"stat": celpy.celtypes.FunctionType}) ] env = mock_cel_environment_integer.return_value assert env.compile.mock_calls == [call("339629869*11")] - assert env.program.mock_calls == [call(sentinel.AST)] + assert env.program.mock_calls == [call(sentinel.AST, functions={'stat': ANY})] prgm = env.program.return_value assert prgm.evaluate.mock_calls == [ call({'jq': celtypes.MapType({celtypes.StringType('name'): celtypes.StringType('CEL')})}) @@ -331,7 +340,7 @@ def test_main_slurp_int_format(mock_cel_environment_integer, caplog, capsys): assert out == "0xdeadbeef\n" assert err == "" -@fixture +@pytest.fixture def mock_cel_environment_bool(monkeypatch): mock_runner = Mock(evaluate=Mock(return_value=celtypes.BoolType(False))) mock_env = Mock( @@ -354,11 +363,11 @@ def test_main_slurp_bool_status(mock_cel_environment_bool, caplog, capsys): sys.stdin = sys.__stdin__ assert status == 1 assert mock_cel_environment_bool.mock_calls == [ - call(package='jq', annotations=None) + call(package='jq', annotations={"stat": celpy.celtypes.FunctionType}) ] env = mock_cel_environment_bool.return_value assert env.compile.mock_calls == [call('.name == "not CEL"')] - assert env.program.mock_calls == [call(sentinel.AST)] + assert env.program.mock_calls == [call(sentinel.AST, functions={'stat': ANY})] prgm = env.program.return_value assert prgm.evaluate.mock_calls == [ call({'jq': celtypes.MapType({celtypes.StringType('name'): celtypes.StringType('CEL')})}) @@ -379,11 +388,13 @@ def test_main_0_int_format(mock_cel_environment_integer, caplog, capsys): status = celpy.__main__.main(argv) assert status == 0 assert mock_cel_environment_integer.mock_calls == [ - call(package=None, annotations=None) + call(package=None, annotations={'stat': celpy.celtypes.FunctionType}) ] env = mock_cel_environment_integer.return_value assert env.compile.mock_calls == [call("339629869*11")] - assert env.program.mock_calls == [call(sentinel.AST)] + assert env.program.mock_calls == [ + call(sentinel.AST, functions={"stat": ANY}) + ] prgm = env.program.return_value assert prgm.evaluate.mock_calls == [call({})] assert caplog.messages == [] @@ -396,7 +407,9 @@ def test_main_verbose(mock_cel_environment, caplog, capsys): argv = ["-v", "[2, 4, 5].map(x, x/2)"] status = celpy.__main__.main(argv) assert status == 0 - assert mock_cel_environment.mock_calls == [call(annotations=None, package="jq")] + assert mock_cel_environment.mock_calls == [ + call(package="jq", annotations={'stat': celpy.celtypes.FunctionType}) + ] assert caplog.messages == ["Expr: '[2, 4, 5].map(x, x/2)'"] out, err = capsys.readouterr() assert out == "" @@ -408,7 +421,9 @@ def test_main_very_verbose(mock_cel_environment, caplog, capsys): argv = ["-vv", "[2, 4, 5].map(x, x/2)"] status = celpy.__main__.main(argv) assert status == 0 - assert mock_cel_environment.mock_calls == [call(annotations=None, package="jq")] + assert mock_cel_environment.mock_calls == [ + call(package="jq", annotations={'stat': celpy.celtypes.FunctionType}) + ] expected_namespace = argparse.Namespace( verbose=2, arg=None, null_input=False, slurp=False, interactive=False, package='jq', document=None, @@ -424,7 +439,7 @@ def test_main_very_verbose(mock_cel_environment, caplog, capsys): assert err == "" -@fixture +@pytest.fixture def mock_cel_environment_syntax_error(monkeypatch): mock_runner = Mock(evaluate=Mock(return_value=str(sentinel.OUTPUT))) mock_env = Mock( @@ -442,7 +457,7 @@ def test_main_parse_error(mock_cel_environment_syntax_error, caplog, capsys): status = celpy.__main__.main(argv) assert status == 1 assert mock_cel_environment_syntax_error.mock_calls == [ - call(package=None, annotations=None) + call(package=None, annotations={'stat': celpy.celtypes.FunctionType}) ] expected_namespace = argparse.Namespace( verbose=0, arg=None, null_input=True, slurp=False, interactive=False, @@ -459,7 +474,7 @@ def test_main_parse_error(mock_cel_environment_syntax_error, caplog, capsys): assert err == "sentinel.Formatted_Error\n" -@fixture +@pytest.fixture def mock_cel_environment_eval_error(monkeypatch): mock_runner = Mock( evaluate=Mock(side_effect=celpy.CELEvalError((sentinel.arg0, sentinel.arg1))) @@ -480,7 +495,7 @@ def test_main_0_eval_error(mock_cel_environment_eval_error, caplog, capsys): status = celpy.__main__.main(argv) assert status == 2 assert mock_cel_environment_eval_error.mock_calls == [ - call(package=None, annotations=None) + call(package=None, annotations={'stat': celpy.celtypes.FunctionType}) ] expected_namespace = argparse.Namespace( verbose=0, arg=None, null_input=True, slurp=False, interactive=False, @@ -505,7 +520,7 @@ def test_main_pipe_eval_error(mock_cel_environment_eval_error, caplog, capsys): sys.stdin = sys.__stdin__ assert status == 0 assert mock_cel_environment_eval_error.mock_calls == [ - call(package="jq", annotations=None) + call(package='jq', annotations={'stat': celpy.celtypes.FunctionType}) ] expected_namespace = argparse.Namespace( verbose=0, arg=None, null_input=False, slurp=False, interactive=False, @@ -531,7 +546,7 @@ def test_main_pipe_json_error(mock_cel_environment_eval_error, caplog, capsys): sys.stdin = sys.__stdin__ assert status == 3 assert mock_cel_environment_eval_error.mock_calls == [ - call(package="jq", annotations=None) + call(package='jq', annotations={'stat': celpy.celtypes.FunctionType}) ] expected_namespace = argparse.Namespace( verbose=0, arg=None, null_input=False, slurp=False, interactive=False, @@ -566,7 +581,7 @@ def test_main_repl(monkeypatch, capsys): def test_repl_class_good_interaction(capsys): """ - If any print() is added for debugging, this test is likely to break. + If any print() is added for debugging, this test will break. """ c = celpy.__main__.CEL_REPL() c.preloop() @@ -581,6 +596,7 @@ def test_repl_class_good_interaction(capsys): r_2 = c.onecmd("quit") assert r_2 out, err = capsys.readouterr() + print(out) # Needed to reveal debugging print() output. lines = out.splitlines() assert lines[0].startswith("3.14159") assert lines[1].startswith("{'pi': DoubleType(3.14159") @@ -604,3 +620,33 @@ def test_repl_class_bad_interaction(capsys): " | ....^" ) assert c.state == {} + + +def test_stat_good(): + cwd = Path.cwd() + doc = celpy.__main__.stat(str(cwd)) + assert doc['st_atime'] == celtypes.TimestampType( + datetime.datetime.fromtimestamp( + cwd.stat().st_atime)) + assert doc['st_ctime'] == celtypes.TimestampType( + datetime.datetime.fromtimestamp( + cwd.stat().st_ctime)) + assert doc['st_mtime'] == celtypes.TimestampType( + datetime.datetime.fromtimestamp( + cwd.stat().st_mtime)) + # Not on all versions of Python. + # assert doc['st_birthtime'] == celtypes.TimestampType( + # datetime.datetime.fromtimestamp( + # cwd.stat().st_birthtime)) + assert doc['st_ino'] == celtypes.IntType(cwd.stat().st_ino) + assert doc['st_size'] == celtypes.IntType(cwd.stat().st_size) + assert doc['st_nlink'] == celtypes.IntType(cwd.stat().st_nlink) + assert doc['kind'] == 'd' + assert doc['setuid'] == celtypes.BoolType(os_stat.S_ISUID & cwd.stat().st_mode != 0) + assert doc['setgid'] == celtypes.BoolType(os_stat.S_ISGID & cwd.stat().st_mode != 0) + assert doc['sticky'] == celtypes.BoolType(os_stat.S_ISVTX & cwd.stat().st_mode != 0) + +def test_stat_does_not_exist(): + path = Path.cwd() / "does_not_exist.tmp" + doc = celpy.__main__.stat(str(path)) + assert doc is None diff --git a/tests/test_package.py b/tests/test_package.py index 6b74c83..77c22e4 100644 --- a/tests/test_package.py +++ b/tests/test_package.py @@ -20,10 +20,10 @@ import json from unittest.mock import Mock, call, sentinel -from pytest import * +import pytest +import lark import celpy -from celpy import celtypes def test_json_to_cel(): @@ -35,19 +35,19 @@ def test_json_to_cel(): {"string": 'embedded "quote"'}, ] actual = celpy.json_to_cel(doc) - expected = celtypes.ListType( + expected = celpy.celtypes.ListType( [ - celtypes.MapType({celtypes.StringType("bool"): celtypes.BoolType(True)}), - celtypes.MapType( + celpy.celtypes.MapType({celpy.celtypes.StringType("bool"): celpy.celtypes.BoolType(True)}), + celpy.celtypes.MapType( { - celtypes.StringType("numbers"): celtypes.ListType( - [celtypes.DoubleType(2.71828), celtypes.IntType(42)] + celpy.celtypes.StringType("numbers"): celpy.celtypes.ListType( + [celpy.celtypes.DoubleType(2.71828), celpy.celtypes.IntType(42)] ) } ), - celtypes.MapType({celtypes.StringType("null"): None}), - celtypes.MapType( - {celtypes.StringType("string"): celtypes.StringType('embedded "quote"')} + celpy.celtypes.MapType({celpy.celtypes.StringType("null"): None}), + celpy.celtypes.MapType( + {celpy.celtypes.StringType("string"): celpy.celtypes.StringType('embedded "quote"')} ), ] ) @@ -57,24 +57,24 @@ def test_json_to_cel(): def test_json_to_cel_unexpected(): """GIVEN JSON doc with invalid type; WHEN json_to_cel(); THEN exception raised.""" doc = {"bytes": b"Ynl0ZXM="} - with raises(ValueError): + with pytest.raises(ValueError): actual = celpy.json_to_cel(doc) def test_encoder(): - cel_obj = celtypes.MapType( + cel_obj = celpy.celtypes.MapType( { - celtypes.StringType("bool"): celtypes.BoolType(True), - celtypes.StringType("numbers"): - celtypes.ListType([ - celtypes.DoubleType(2.71828), celtypes.UintType(42) + celpy.celtypes.StringType("bool"): celpy.celtypes.BoolType(True), + celpy.celtypes.StringType("numbers"): + celpy.celtypes.ListType([ + celpy.celtypes.DoubleType(2.71828), celpy.celtypes.UintType(42) ]), - celtypes.StringType("null"): None, - celtypes.StringType("string"): celtypes.StringType('embedded "quote"'), - celtypes.StringType("bytes"): - celtypes.BytesType(bytes([0x62, 0x79, 0x74, 0x65, 0x73])), - celtypes.StringType("timestamp"): celtypes.TimestampType('2009-02-13T23:31:30Z'), - celtypes.StringType("duration"): celtypes.DurationType('42s'), + celpy.celtypes.StringType("null"): None, + celpy.celtypes.StringType("string"): celpy.celtypes.StringType('embedded "quote"'), + celpy.celtypes.StringType("bytes"): + celpy.celtypes.BytesType(bytes([0x62, 0x79, 0x74, 0x65, 0x73])), + celpy.celtypes.StringType("timestamp"): celpy.celtypes.TimestampType('2009-02-13T23:31:30Z'), + celpy.celtypes.StringType("duration"): celpy.celtypes.DurationType('42s'), } ) json_text = json.dumps(cel_obj, cls=celpy.CELJSONEncoder) @@ -86,7 +86,7 @@ def test_encoder(): def test_encoder_unknown(): cel_obj = sentinel.no_json - with raises(TypeError): + with pytest.raises(TypeError): json_text = json.dumps(cel_obj, cls=celpy.CELJSONEncoder) @@ -97,72 +97,67 @@ def test_decoder(): '"timestamp": "2009-02-13T23:31:30Z", "duration": "42s"}' ) cel_obj = json.loads(json_text, cls=celpy.CELJSONDecoder) - assert cel_obj == celtypes.MapType({ - celtypes.StringType('bool'): celtypes.IntType(1), - celtypes.StringType('bytes'): celtypes.StringType('Ynl0ZXM='), - celtypes.StringType('duration'): celtypes.StringType('42s'), - celtypes.StringType('null'): None, - celtypes.StringType('numbers'): - celtypes.ListType([celtypes.DoubleType(2.71828), celtypes.IntType(42)]), - celtypes.StringType('string'): celtypes.StringType('embedded "quote"'), - celtypes.StringType('timestamp'): celtypes.StringType('2009-02-13T23:31:30Z'), + assert cel_obj == celpy.celtypes.MapType({ + celpy.celtypes.StringType('bool'): celpy.celtypes.IntType(1), + celpy.celtypes.StringType('bytes'): celpy.celtypes.StringType('Ynl0ZXM='), + celpy.celtypes.StringType('duration'): celpy.celtypes.StringType('42s'), + celpy.celtypes.StringType('null'): None, + celpy.celtypes.StringType('numbers'): + celpy.celtypes.ListType([celpy.celtypes.DoubleType(2.71828), celpy.celtypes.IntType(42)]), + celpy.celtypes.StringType('string'): celpy.celtypes.StringType('embedded "quote"'), + celpy.celtypes.StringType('timestamp'): celpy.celtypes.StringType('2009-02-13T23:31:30Z'), }) -@fixture -def mock_evaluator(monkeypatch): - evaluator = Mock(evaluate=Mock(return_value=sentinel.Output)) - evaluator_class = Mock(return_value=evaluator) - monkeypatch.setattr(celpy, "Evaluator", evaluator_class) - return evaluator_class - - -@fixture +@pytest.fixture def mock_environment(monkeypatch): environment = Mock( - activation=Mock( - return_value=Mock(nested_activation=Mock(return_value=sentinel.Activation)) - ) + package=sentinel.Package, + annotations={}, ) return environment -def test_interp_runner(mock_evaluator, mock_environment): +def test_interp_runner(mock_environment): """ GIVEN Environment and AST and mocked Evaluator WHEN InterpretedRunner created and evaluated THEN Runner uses Environment, AST, and the mocked Evaluator """ - functions = [sentinel.Function] - r = celpy.InterpretedRunner(mock_environment, sentinel.AST, functions) + def a_function(): + return None + functions = [a_function] + ast = Mock(spec=lark.Tree, children=[lark.Token(type_="BOOL_LIT", value="true"),], data="literal") + r = celpy.InterpretedRunner(mock_environment, ast, functions) result = r.evaluate({"variable": sentinel.variable}) - assert result == sentinel.Output - - assert mock_evaluator.mock_calls == [ - call( - activation=mock_environment.activation.return_value.nested_activation.return_value, - ast=sentinel.AST, - functions=[sentinel.Function], - ) - ] - assert mock_evaluator.return_value.evaluate.mock_calls == [call()] + assert result == celpy.celtypes.BoolType(True) + + +@pytest.fixture +def mock_ast(): + # Reset the ClassVar CEL_PARSER. + celpy.CELParser.CEL_PARSER = None + parser = celpy.CELParser(tree_class=celpy.evaluation.TranspilerTree) + source = "true" + tree = parser.parse(source) + return tree -def test_compiled_runner(mock_evaluator, mock_environment): +def test_compiled_runner(mock_environment, mock_ast): """ GIVEN Environment and AST and mocked Evaluator WHEN InterpretedRunner created and evaluated THEN Runner uses Environment, AST, and the mocked Evaluator - - Currently, the CompiledRunner class is a place-holder implementation. """ - functions = [sentinel.Function] - r = celpy.CompiledRunner(mock_environment, sentinel.AST, functions) - with raises(NotImplementedError): - result = r.evaluate({"variable": sentinel.variable}) - + def a_function(): + return None + functions = [a_function] + r = celpy.CompiledRunner(mock_environment, mock_ast, functions) + assert r.tp.source_text.strip() == "CEL = celpy.evaluation.result(base_activation, lambda activation: celpy.celtypes.BoolType(True))" + result = r.evaluate({"variable": sentinel.variable}) + assert result == celpy.celtypes.BoolType(True) -@fixture +@pytest.fixture def mock_parser(monkeypatch): parser = Mock(parse=Mock(return_value=sentinel.AST)) parser_class = Mock(return_value=parser) @@ -170,7 +165,7 @@ def mock_parser(monkeypatch): return parser_class -@fixture +@pytest.fixture def mock_runner(monkeypatch): runner = Mock() runner_class = Mock(return_value=runner) @@ -178,7 +173,7 @@ def mock_runner(monkeypatch): return runner_class -@fixture +@pytest.fixture def mock_activation(monkeypatch): activation = Mock() activation_class = Mock(return_value=activation) @@ -187,7 +182,7 @@ def mock_activation(monkeypatch): def test_environment(mock_parser, mock_runner, mock_activation): - e = celpy.Environment(sentinel.package, {sentinel.variable: celtypes.UintType}) + e = celpy.Environment(sentinel.package, {sentinel.variable: celpy.celtypes.UintType}) ast = e.compile(sentinel.Source) assert ast == sentinel.AST assert mock_parser.return_value.parse.mock_calls == [call(sentinel.Source)] @@ -195,15 +190,19 @@ def test_environment(mock_parser, mock_runner, mock_activation): pgm = e.program(ast, functions=[sentinel.Function]) assert pgm == mock_runner.return_value assert mock_runner.mock_calls == [call(e, sentinel.AST, [sentinel.Function])] - act = e.activation() - assert act == mock_activation.return_value - expected = { - sentinel.variable: celtypes.UintType, - } - expected.update(celpy.googleapis) - assert mock_activation.mock_calls == [ - call( - annotations=expected, - package=sentinel.package - ) - ] + assert e.annotations[sentinel.variable] == celpy.celtypes.UintType + + # OLD DESIGN + # act = e.activation() + # assert act == mock_activation.return_value + # expected = { + # sentinel.variable: celtypes.UintType, + # } + # TESTS Activation, doesn't really belong here + # expected.update(celpy.googleapis) + # assert mock_activation.mock_calls == [ + # call( + # annotations=expected, + # package=sentinel.package + # ) + # ] diff --git a/tests/test_transpilation.py b/tests/test_transpilation.py new file mode 100644 index 0000000..f8d70eb --- /dev/null +++ b/tests/test_transpilation.py @@ -0,0 +1,663 @@ +# SPDX-Copyright: Copyright (c) Capital One Services, LLC +# SPDX-License-Identifier: Apache-2.0 +# Copyright 2020 Capital One Services, LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and limitations under the License. +""" +Test all the transpilation methods. + +A large number of tests use :py:meth:`Transpiler.transpile`. + +The approach used here may not be ideal from a unit testing perspective. +This approach tends to test the superclass :py:meth:`lark.visitors.Visitor_Recursive.visit`, +which involves re-testing a fair amount of Lark code. + +Further, we don't follow the path used by the :py:mod:`test_evaluation` test suite. +This module does not synthesize parse trees to disentangle itself from the parser. +Instead, it parses CEL expressions directly. + +This does not precisely parallel :py:mod:`test_evaluation`. +The :py:mod:`test_evaluation` module tests a number of elements of evaluation +that are outside the Evaluator, including `Activation`, `Referent`, `NameContainer`. +This module cherry-picks tests of CEL expressions separate from the evaluation mechanics. + +""" +import ast +from textwrap import dedent +from types import SimpleNamespace +from unittest.mock import Mock, MagicMock, sentinel + +import lark +import pytest + +import celpy.evaluation # Expose the name for monkeypatching +from celpy import celparser, celtypes +from celpy.evaluation import * + + +@pytest.fixture +def mock_protobuf(): + """Used for a few test cases.""" + def get_method(name, default=None): + if name == "field": + return 42 + else: + raise KeyError(name) + protobuf_message = Mock(name="protobuf_message", spec=celtypes.MessageType, get=Mock(side_effect=get_method)) + protobuf_message_class = Mock(name="protobuf_message class", return_value=protobuf_message) + return protobuf_message_class + +# This may be slightly better for isolating Activation implementation. +# @pytest.fixture +# def mock_activation(): +# activation = Mock( +# resolve_name=Mock(return_value=lambda name: {"name2": celtypes.IntType}.get()) +# ) +# return activation + +@pytest.fixture +def mock_functions(): + return {"no_arg_function": no_arg_function} + + +@pytest.fixture +def mock_activation(mock_protobuf, mock_functions): + """ + See :py:class:`NameContainer`, specifically :py:meth:`NameContainer.load_annotations` + """ + return Activation( + annotations={ + "name1.name2": celtypes.IntType, + "protobuf_message": mock_protobuf, + "a.b.c": celtypes.StringType, + }, + functions=mock_functions, + vars={ + "duration": celtypes.DurationType(seconds=123, nanos=123456789), + "a.b.c": celtypes.StringType("yeah"), + } + ) + + +def no_arg_function(): + return celpy.celtypes.IntType(42) + +@pytest.fixture +def mock_globals(mock_activation, mock_protobuf): + # Works, but feels sketchy... + # We're tweaking one function's understanding of globals. + global_vars = celpy.evaluation.result.__globals__ + global_vars["the_activation"] = mock_activation + global_vars["protobuf_message"] = mock_protobuf + global_vars["test_transpilation"] = SimpleNamespace(no_arg_function=no_arg_function) + + # Seems to make more sense, but doesn't actually work! + # global the_activation + # the_activation = mock_activation + # global_vars = globals().copy() + return global_vars + +def test_result_builder(mock_globals, mock_activation): + def mock_operation(a, b): + if isinstance(a, Exception): + raise a + else: + return sentinel.A_OP_B + + + expr_1 = lambda activation: mock_operation(TypeError(sentinel.type_error_message), sentinel.VALUE) + result_1 = celpy.evaluation.result(mock_activation, expr_1) + assert isinstance(result_1, CELEvalError) + assert result_1.args == ('no such overload', TypeError, (sentinel.type_error_message,)) + assert result_1.__cause__.__class__ == TypeError + + with pytest.raises(IOError) as exc_info: + expr_2 = lambda activation: mock_operation(IOError(sentinel.io_error_message), sentinel.VALUE) + result_2 = celpy.evaluation.result(mock_activation, expr_2) + assert not isinstance(exc_info.value, CELEvalError) + assert exc_info.value.args == (sentinel.io_error_message,) + + result_3 = mock_operation(sentinel.A, sentinel.B) + assert result_3 == sentinel.A_OP_B + + +literals = [ + ('3.14', + "CEL = celpy.evaluation.result(base_activation, lambda activation: celpy.celtypes.DoubleType(3.14))", + celpy.celtypes.DoubleType(3.14), + "literal"), + ('42', + "CEL = celpy.evaluation.result(base_activation, lambda activation: celpy.celtypes.IntType(42))", + celpy.celtypes.IntType(42), + "literal"), + ('42u', + "CEL = celpy.evaluation.result(base_activation, lambda activation: celpy.celtypes.UintType(42))", + celpy.celtypes.UintType(42), + "literal"), + ('b"B"', + "CEL = celpy.evaluation.result(base_activation, lambda activation: celpy.celtypes.BytesType(b'B'))", + celpy.celtypes.BytesType(b'B'), + "literal"), + ('"String"', + "CEL = celpy.evaluation.result(base_activation, lambda activation: celpy.celtypes.StringType('String'))", + celpy.celtypes.StringType("String"), + "literal"), + ('true', + "CEL = celpy.evaluation.result(base_activation, lambda activation: celpy.celtypes.BoolType(True))", + celpy.celtypes.BoolType(True), + "literal"), + ('null', + "CEL = celpy.evaluation.result(base_activation, lambda activation: None)", + None, # celpy.celtypes.NullType(), + "literal"), + ('[]', + "CEL = celpy.evaluation.result(base_activation, lambda activation: celpy.celtypes.ListType([]))", + celpy.celtypes.ListType([]), + "literal"), + ('{}', + "CEL = celpy.evaluation.result(base_activation, lambda activation: celpy.celtypes.MapType([]))", + celpy.celtypes.MapType({}), + "literal"), + ('bool', + "CEL = celpy.evaluation.result(base_activation, lambda activation: activation.bool)", + celpy.celtypes.BoolType, + "literal"), +] + +function_params = [ + ("size([42, 6, 7])", + "CEL = celpy.evaluation.result(base_activation, lambda activation: celpy.evaluation.function_size(celpy.celtypes.ListType([celpy.celtypes.IntType(42), celpy.celtypes.IntType(6), celpy.celtypes.IntType(7)])))", + celpy.celtypes.IntType(3), + "IDENT(_)"), + ("size(3.14)", + "CEL = celpy.evaluation.result(base_activation, lambda activation: celpy.evaluation.function_size(celpy.celtypes.DoubleType(3.14)))", + CELEvalError, + "IDENT(_)"), + ('"hello".size()', + "CEL = celpy.evaluation.result(base_activation, lambda activation: celpy.evaluation.function_size(celpy.celtypes.StringType('hello')))", + celpy.celtypes.IntType(5), + "_.IDENT()"), +] + +method_params = [ + ("[42, 6, 7].size()", + "CEL = celpy.evaluation.result(base_activation, lambda activation: celpy.evaluation.function_size(celpy.celtypes.ListType([celpy.celtypes.IntType(42), celpy.celtypes.IntType(6), celpy.celtypes.IntType(7)])))", + celpy.celtypes.IntType(3), + "_.size()"), + ('timestamp("2009-02-13T23:31:30Z").getMonth()', + "CEL = celpy.evaluation.result(base_activation, lambda activation: celpy.evaluation.function_getMonth(celpy.celtypes.TimestampType(celpy.celtypes.StringType('2009-02-13T23:31:30Z'))))", + celtypes.IntType(1), + "_._())"), + ('["hello", "world"].contains("hello")', + "CEL = celpy.evaluation.result(base_activation, lambda activation: celpy.evaluation.function_contains(celpy.celtypes.ListType([celpy.celtypes.StringType('hello'), celpy.celtypes.StringType('world')]), celpy.celtypes.StringType('hello')))", + celtypes.BoolType(True), + "_._(_)"), +] + +macro_has_params = [ + ('has({"n": 355, "d": 113}.n)', + dedent("""\ + # ident_arg has: + ex_9_h = lambda activation: celpy.celtypes.MapType([(celpy.celtypes.StringType('n'), celpy.celtypes.IntType(355)), (celpy.celtypes.StringType('d'), celpy.celtypes.IntType(113))]).get('n') + ex_9 = lambda activation: not isinstance(celpy.evaluation.result(activation, ex_9_h), CELEvalError) + CEL = celpy.evaluation.result(base_activation, ex_9)"""), + celtypes.BoolType(True), + "has(_._)"), + ('has({"n": 355, "d": 113}.nope)', + dedent("""\ + # ident_arg has: + ex_9_h = lambda activation: celpy.celtypes.MapType([(celpy.celtypes.StringType('n'), celpy.celtypes.IntType(355)), (celpy.celtypes.StringType('d'), celpy.celtypes.IntType(113))]).get('nope') + ex_9 = lambda activation: not isinstance(celpy.evaluation.result(activation, ex_9_h), CELEvalError) + CEL = celpy.evaluation.result(base_activation, ex_9)""" + ), + celtypes.BoolType(False), + "has(_._)"), + ('dyn(6) * 7', + "CEL = celpy.evaluation.result(base_activation, lambda activation: operator.mul(celpy.celtypes.IntType(6), celpy.celtypes.IntType(7)))", + celtypes.IntType(42), + "dyn(_)"), + ("type(dyn([1, 'one']))", + "CEL = celpy.evaluation.result(base_activation, lambda activation: celpy.celtypes.TypeType(celpy.celtypes.ListType([celpy.celtypes.IntType(1), celpy.celtypes.StringType('one')])))", + celtypes.ListType, + "dyn(_)"), +] + +unary_operator_params = [ + ("! true", "CEL = celpy.evaluation.result(base_activation, lambda activation: celpy.celtypes.logical_not(celpy.celtypes.BoolType(True)))", celtypes.BoolType(False), "!_"), + ("- 42", "CEL = celpy.evaluation.result(base_activation, lambda activation: operator.neg(celpy.celtypes.IntType(42)))", celtypes.IntType(-42), "-_"), + ("- -9223372036854775808", "CEL = celpy.evaluation.result(base_activation, lambda activation: operator.neg(celpy.celtypes.IntType(-9223372036854775808)))", CELEvalError, "-_"), +] + +binary_operator_params = [ + ("6 < 7", "CEL = celpy.evaluation.result(base_activation, lambda activation: celpy.evaluation.bool_lt(celpy.celtypes.IntType(6), celpy.celtypes.IntType(7)))", celtypes.BoolType(True), "_<_"), + ("6 <= 7", "CEL = celpy.evaluation.result(base_activation, lambda activation: celpy.evaluation.bool_le(celpy.celtypes.IntType(6), celpy.celtypes.IntType(7)))", celtypes.BoolType(True), "_<=_"), + ("6 > 7", "CEL = celpy.evaluation.result(base_activation, lambda activation: celpy.evaluation.bool_gt(celpy.celtypes.IntType(6), celpy.celtypes.IntType(7)))", celtypes.BoolType(False), "_>_"), + ("6 >= 7", "CEL = celpy.evaluation.result(base_activation, lambda activation: celpy.evaluation.bool_ge(celpy.celtypes.IntType(6), celpy.celtypes.IntType(7)))", celtypes.BoolType(False), "_>=_"), + ("42 == 42", "CEL = celpy.evaluation.result(base_activation, lambda activation: celpy.evaluation.bool_eq(celpy.celtypes.IntType(42), celpy.celtypes.IntType(42)))", celtypes.BoolType(True), "_==_"), + ("[] == []", "CEL = celpy.evaluation.result(base_activation, lambda activation: celpy.evaluation.bool_eq(celpy.celtypes.ListType([]), celpy.celtypes.ListType([])))", celtypes.BoolType(True), "_==_"), + ("42 != 42", "CEL = celpy.evaluation.result(base_activation, lambda activation: celpy.evaluation.bool_ne(celpy.celtypes.IntType(42), celpy.celtypes.IntType(42)))", celtypes.BoolType(False), "_!=_"), + ('"b" in ["a", "b", "c"]', + "CEL = celpy.evaluation.result(base_activation, lambda activation: celpy.evaluation.operator_in(celpy.celtypes.StringType('b'), celpy.celtypes.ListType([celpy.celtypes.StringType('a'), celpy.celtypes.StringType('b'), celpy.celtypes.StringType('c')])))", + celtypes.BoolType(True), + "_in_"), + ("40 + 2", "CEL = celpy.evaluation.result(base_activation, lambda activation: operator.add(celpy.celtypes.IntType(40), celpy.celtypes.IntType(2)))", celtypes.IntType(42), "_+_"), + ("44 - 2", "CEL = celpy.evaluation.result(base_activation, lambda activation: operator.sub(celpy.celtypes.IntType(44), celpy.celtypes.IntType(2)))", celtypes.IntType(42), "_-_"), + ("6 * 7", "CEL = celpy.evaluation.result(base_activation, lambda activation: operator.mul(celpy.celtypes.IntType(6), celpy.celtypes.IntType(7)))", celtypes.IntType(42), "_*_"), + ("84 / 2", "CEL = celpy.evaluation.result(base_activation, lambda activation: operator.truediv(celpy.celtypes.IntType(84), celpy.celtypes.IntType(2)))", celtypes.IntType(42), "_/_"), + ("85 % 43", "CEL = celpy.evaluation.result(base_activation, lambda activation: operator.mod(celpy.celtypes.IntType(85), celpy.celtypes.IntType(43)))", celtypes.IntType(42), "_%_"), + # A few error examples + ('42 in ["a", "b", "c"]', + "CEL = celpy.evaluation.result(base_activation, lambda activation: celpy.evaluation.operator_in(celpy.celtypes.IntType(42), celpy.celtypes.ListType([celpy.celtypes.StringType('a'), celpy.celtypes.StringType('b'), celpy.celtypes.StringType('c')])))", + CELEvalError, + "_in_"), + ("9223372036854775807 + 1", "CEL = celpy.evaluation.result(base_activation, lambda activation: operator.add(celpy.celtypes.IntType(9223372036854775807), celpy.celtypes.IntType(1)))", CELEvalError, "_+_"), + ("9223372036854775807 * 2", "CEL = celpy.evaluation.result(base_activation, lambda activation: operator.mul(celpy.celtypes.IntType(9223372036854775807), celpy.celtypes.IntType(2)))", CELEvalError, "_*_"), + ("84 / 0", "CEL = celpy.evaluation.result(base_activation, lambda activation: operator.truediv(celpy.celtypes.IntType(84), celpy.celtypes.IntType(0)))", CELEvalError, "_/_"), +] + +short_circuit_params = [ + ("true || (3 / 0 != 0)", + dedent("""\ + # conditionalor: + ex_1_l = lambda activation: celpy.celtypes.BoolType(True) + ex_1_r = lambda activation: celpy.evaluation.bool_ne(operator.truediv(celpy.celtypes.IntType(3), celpy.celtypes.IntType(0)), celpy.celtypes.IntType(0)) + ex_1 = lambda activation: celpy.celtypes.logical_or(celpy.evaluation.result(activation, ex_1_l), celpy.evaluation.result(activation, ex_1_r)) + CEL = celpy.evaluation.result(base_activation, ex_1)"""), + celtypes.BoolType(True), "_||_"), + ("(3 / 0 != 0) || true", + dedent("""\ + # conditionalor: + ex_1_l = lambda activation: celpy.evaluation.bool_ne(operator.truediv(celpy.celtypes.IntType(3), celpy.celtypes.IntType(0)), celpy.celtypes.IntType(0)) + ex_1_r = lambda activation: celpy.celtypes.BoolType(True) + ex_1 = lambda activation: celpy.celtypes.logical_or(celpy.evaluation.result(activation, ex_1_l), celpy.evaluation.result(activation, ex_1_r)) + CEL = celpy.evaluation.result(base_activation, ex_1)"""), + celtypes.BoolType(True), "_||_"), + ("false || (3 / 0 != 0)", + dedent("""\ + # conditionalor: + ex_1_l = lambda activation: celpy.celtypes.BoolType(False) + ex_1_r = lambda activation: celpy.evaluation.bool_ne(operator.truediv(celpy.celtypes.IntType(3), celpy.celtypes.IntType(0)), celpy.celtypes.IntType(0)) + ex_1 = lambda activation: celpy.celtypes.logical_or(celpy.evaluation.result(activation, ex_1_l), celpy.evaluation.result(activation, ex_1_r)) + CEL = celpy.evaluation.result(base_activation, ex_1)"""), + CELEvalError, "_||_"), + ("(3 / 0 != 0) || false", + dedent("""\ + # conditionalor: + ex_1_l = lambda activation: celpy.evaluation.bool_ne(operator.truediv(celpy.celtypes.IntType(3), celpy.celtypes.IntType(0)), celpy.celtypes.IntType(0)) + ex_1_r = lambda activation: celpy.celtypes.BoolType(False) + ex_1 = lambda activation: celpy.celtypes.logical_or(celpy.evaluation.result(activation, ex_1_l), celpy.evaluation.result(activation, ex_1_r)) + CEL = celpy.evaluation.result(base_activation, ex_1)"""), + CELEvalError, "_||_"), + + ("true && 3 / 0", + dedent("""\ + # conditionaland: + ex_2_l = lambda activation: celpy.celtypes.BoolType(True) + ex_2_r = lambda activation: operator.truediv(celpy.celtypes.IntType(3), celpy.celtypes.IntType(0)) + ex_2 = lambda activation: celpy.celtypes.logical_and(celpy.evaluation.result(activation, ex_2_l), celpy.evaluation.result(activation, ex_2_r)) + CEL = celpy.evaluation.result(base_activation, ex_2)"""), + CELEvalError, "_&&_"), + ("false && 3 / 0", + dedent("""\ + # conditionaland: + ex_2_l = lambda activation: celpy.celtypes.BoolType(False) + ex_2_r = lambda activation: operator.truediv(celpy.celtypes.IntType(3), celpy.celtypes.IntType(0)) + ex_2 = lambda activation: celpy.celtypes.logical_and(celpy.evaluation.result(activation, ex_2_l), celpy.evaluation.result(activation, ex_2_r)) + CEL = celpy.evaluation.result(base_activation, ex_2)"""), + celpy.celtypes.BoolType(False), "_&&_"), + ("3 / 0 && true", + dedent("""\ + # conditionaland: + ex_2_l = lambda activation: operator.truediv(celpy.celtypes.IntType(3), celpy.celtypes.IntType(0)) + ex_2_r = lambda activation: celpy.celtypes.BoolType(True) + ex_2 = lambda activation: celpy.celtypes.logical_and(celpy.evaluation.result(activation, ex_2_l), celpy.evaluation.result(activation, ex_2_r)) + CEL = celpy.evaluation.result(base_activation, ex_2)"""), + CELEvalError, "_&&_"), + ("3 / 0 && false", + dedent("""\ + # conditionaland: + ex_2_l = lambda activation: operator.truediv(celpy.celtypes.IntType(3), celpy.celtypes.IntType(0)) + ex_2_r = lambda activation: celpy.celtypes.BoolType(False) + ex_2 = lambda activation: celpy.celtypes.logical_and(celpy.evaluation.result(activation, ex_2_l), celpy.evaluation.result(activation, ex_2_r)) + CEL = celpy.evaluation.result(base_activation, ex_2)"""), + celpy.celtypes.BoolType(False), "_&&_"), + + ("(13 % 2 != 0) ? (13 * 3 + 1) : (13 / 0)", + dedent("""\ + # expr: + ex_0_c = lambda activation: celpy.evaluation.bool_ne(operator.mod(celpy.celtypes.IntType(13), celpy.celtypes.IntType(2)), celpy.celtypes.IntType(0)) + ex_0_l = lambda activation: operator.add(operator.mul(celpy.celtypes.IntType(13), celpy.celtypes.IntType(3)), celpy.celtypes.IntType(1)) + ex_0_r = lambda activation: operator.truediv(celpy.celtypes.IntType(13), celpy.celtypes.IntType(0)) + ex_0 = lambda activation: celpy.celtypes.logical_condition(celpy.evaluation.result(activation, ex_0_c), celpy.evaluation.result(activation, ex_0_l), celpy.evaluation.result(activation, ex_0_r)) + CEL = celpy.evaluation.result(base_activation, ex_0)"""), + celtypes.IntType(40), + "_?_:_"), + ("(12 % 2 != 0) ? (12 / 0) : (12 / 2)", + dedent("""\ + # expr: + ex_0_c = lambda activation: celpy.evaluation.bool_ne(operator.mod(celpy.celtypes.IntType(12), celpy.celtypes.IntType(2)), celpy.celtypes.IntType(0)) + ex_0_l = lambda activation: operator.truediv(celpy.celtypes.IntType(12), celpy.celtypes.IntType(0)) + ex_0_r = lambda activation: operator.truediv(celpy.celtypes.IntType(12), celpy.celtypes.IntType(2)) + ex_0 = lambda activation: celpy.celtypes.logical_condition(celpy.evaluation.result(activation, ex_0_c), celpy.evaluation.result(activation, ex_0_l), celpy.evaluation.result(activation, ex_0_r)) + CEL = celpy.evaluation.result(base_activation, ex_0)"""), + celtypes.IntType(6), + "_?_:_"), + ("(14 % 0 != 0) ? (14 * 3 + 1) : (14 / 2)", + dedent("""\ + # expr: + ex_0_c = lambda activation: celpy.evaluation.bool_ne(operator.mod(celpy.celtypes.IntType(14), celpy.celtypes.IntType(0)), celpy.celtypes.IntType(0)) + ex_0_l = lambda activation: operator.add(operator.mul(celpy.celtypes.IntType(14), celpy.celtypes.IntType(3)), celpy.celtypes.IntType(1)) + ex_0_r = lambda activation: operator.truediv(celpy.celtypes.IntType(14), celpy.celtypes.IntType(2)) + ex_0 = lambda activation: celpy.celtypes.logical_condition(celpy.evaluation.result(activation, ex_0_c), celpy.evaluation.result(activation, ex_0_l), celpy.evaluation.result(activation, ex_0_r)) + CEL = celpy.evaluation.result(base_activation, ex_0)"""), + CELEvalError, + "_?_:_"), +] + +member_dot_params = [ + ('{"field": 42}.field', + dedent("""\ + CEL = celpy.evaluation.result(base_activation, lambda activation: celpy.celtypes.MapType([(celpy.celtypes.StringType('field'), celpy.celtypes.IntType(42))]).get('field'))"""), + celtypes.IntType(42), + "_._"), + # Must match the mock_protobuf message + ('protobuf_message{field: 42}.field', + dedent("""\ + CEL = celpy.evaluation.result(base_activation, lambda activation: activation.protobuf_message([('field', celpy.celtypes.IntType(42))]).get('field'))"""), + celtypes.IntType(42), + "_._"), + # Must NOT match the mock_protobuf message + ('protobuf_message{field: 42}.not_the_name', + dedent("""\ + CEL = celpy.evaluation.result(base_activation, lambda activation: activation.protobuf_message([('field', celpy.celtypes.IntType(42))]).get('not_the_name'))"""), + CELEvalError, + "_._"), + # Requires mock_activation with {"name1.name2": celtypes.IntType} + ('name1.name2', + dedent("""\ + CEL = celpy.evaluation.result(base_activation, lambda activation: activation.name1.get('name2'))"""), + celtypes.IntType, + "_._"), + ('a.b.c', + dedent("""\ + CEL = celpy.evaluation.result(base_activation, lambda activation: activation.a.get('b').get('c'))"""), + celtypes.StringType("yeah"), + "_._"), +] + +member_item_params = [ + ('["hello", "world"][0]', + "CEL = celpy.evaluation.result(base_activation, lambda activation: operator.getitem(celpy.celtypes.ListType([celpy.celtypes.StringType('hello'), celpy.celtypes.StringType('world')]), celpy.celtypes.IntType(0)))", + celtypes.StringType("hello"), + "_.[_]"), + ('["hello", "world"][42]', + "CEL = celpy.evaluation.result(base_activation, lambda activation: operator.getitem(celpy.celtypes.ListType([celpy.celtypes.StringType('hello'), celpy.celtypes.StringType('world')]), celpy.celtypes.IntType(42)))", + CELEvalError, + "_.[_]"), + ('["hello", "world"][3.14]', + "CEL = celpy.evaluation.result(base_activation, lambda activation: operator.getitem(celpy.celtypes.ListType([celpy.celtypes.StringType('hello'), celpy.celtypes.StringType('world')]), celpy.celtypes.DoubleType(3.14)))", + CELEvalError, + "_.[_]"), + ('{"hello": "world"}["hello"]', + "CEL = celpy.evaluation.result(base_activation, lambda activation: operator.getitem(celpy.celtypes.MapType([(celpy.celtypes.StringType('hello'), celpy.celtypes.StringType('world'))]), celpy.celtypes.StringType('hello')))", + celtypes.StringType("world"), + "_.[_]"), + ('{"hello": "world"}["world"]', + "CEL = celpy.evaluation.result(base_activation, lambda activation: operator.getitem(celpy.celtypes.MapType([(celpy.celtypes.StringType('hello'), celpy.celtypes.StringType('world'))]), celpy.celtypes.StringType('world')))", + CELEvalError, + "_.[_]"), +] + +# A protobuf message class, `protobuf_message`, is required in the activation. +member_object_params = [ + # Must match the mock_protobuf fixture + ('protobuf_message{field: 42}', + "CEL = celpy.evaluation.result(base_activation, lambda activation: activation.protobuf_message([('field', celpy.celtypes.IntType(42))]))", + sentinel.MESSAGE, + "_.{_}"), + ('protobuf_message{}', + "CEL = celpy.evaluation.result(base_activation, lambda activation: activation.protobuf_message([]))", + sentinel.MESSAGE, + "_.{_}"), +] + +member_dot_arg_method = [ + ('timestamp("2009-02-13T23:31:30Z").getMonth()', + dedent("""\ + CEL = celpy.evaluation.result(base_activation, lambda activation: celpy.evaluation.function_getMonth(celpy.celtypes.TimestampType(celpy.celtypes.StringType('2009-02-13T23:31:30Z'))))"""), + celtypes.IntType(1), + "_._(_)"), + ('timestamp("2009-02-13T23:31:30Z").getDate()', + dedent("""\ + CEL = celpy.evaluation.result(base_activation, lambda activation: celpy.evaluation.function_getDate(celpy.celtypes.TimestampType(celpy.celtypes.StringType('2009-02-13T23:31:30Z'))))"""), + celtypes.IntType(13), + "_._(_)"), + ('timestamp("2009-02-13T23:31:30Z").getDayOfMonth()', + dedent("""\ + CEL = celpy.evaluation.result(base_activation, lambda activation: celpy.evaluation.function_getDayOfMonth(celpy.celtypes.TimestampType(celpy.celtypes.StringType('2009-02-13T23:31:30Z'))))"""), + celtypes.IntType(12), + "_._(_)"), + ('timestamp("2009-02-13T23:31:30Z").getDayOfWeek()', + dedent("""\ + CEL = celpy.evaluation.result(base_activation, lambda activation: celpy.evaluation.function_getDayOfWeek(celpy.celtypes.TimestampType(celpy.celtypes.StringType('2009-02-13T23:31:30Z'))))"""), + celtypes.IntType(5), + "_._(_)"), + ('timestamp("2009-02-13T23:31:30Z").getDayOfYear()', + dedent("""\ + CEL = celpy.evaluation.result(base_activation, lambda activation: celpy.evaluation.function_getDayOfYear(celpy.celtypes.TimestampType(celpy.celtypes.StringType('2009-02-13T23:31:30Z'))))"""), + celtypes.IntType(43), + "_._(_)"), + ('timestamp("2009-02-13T23:31:30Z").getFullYear()', + dedent("""\ + CEL = celpy.evaluation.result(base_activation, lambda activation: celpy.evaluation.function_getFullYear(celpy.celtypes.TimestampType(celpy.celtypes.StringType('2009-02-13T23:31:30Z'))))"""), + celtypes.IntType(2009), + "_._(_)"), + ('timestamp("2009-02-13T23:31:30Z").getHours()', + dedent("""\ + CEL = celpy.evaluation.result(base_activation, lambda activation: celpy.evaluation.function_getHours(celpy.celtypes.TimestampType(celpy.celtypes.StringType('2009-02-13T23:31:30Z'))))"""), + celtypes.IntType(23), + "_._(_)"), + ('timestamp("2009-02-13T23:31:30Z").getMilliseconds()', + dedent("""\ + CEL = celpy.evaluation.result(base_activation, lambda activation: celpy.evaluation.function_getMilliseconds(celpy.celtypes.TimestampType(celpy.celtypes.StringType('2009-02-13T23:31:30Z'))))"""), + celtypes.IntType(0), + "_._(_)"), + ('timestamp("2009-02-13T23:31:30Z").getMinutes()', + dedent("""\ + CEL = celpy.evaluation.result(base_activation, lambda activation: celpy.evaluation.function_getMinutes(celpy.celtypes.TimestampType(celpy.celtypes.StringType('2009-02-13T23:31:30Z'))))"""), + celtypes.IntType(31), + "_._(_)"), + ('timestamp("2009-02-13T23:31:30Z").getSeconds()', + dedent("""\ + CEL = celpy.evaluation.result(base_activation, lambda activation: celpy.evaluation.function_getSeconds(celpy.celtypes.TimestampType(celpy.celtypes.StringType('2009-02-13T23:31:30Z'))))"""), + celtypes.IntType(30), + "_._(_)"), + ('["hello", "world"].contains("hello")', + dedent("""\ + CEL = celpy.evaluation.result(base_activation, lambda activation: celpy.evaluation.function_contains(celpy.celtypes.ListType([celpy.celtypes.StringType('hello'), celpy.celtypes.StringType('world')]), celpy.celtypes.StringType('hello')))"""), + celtypes.BoolType(True), + "_._(_)"), + ('"hello".startsWith("h")', + dedent("""\ + CEL = celpy.evaluation.result(base_activation, lambda activation: celpy.evaluation.function_startsWith(celpy.celtypes.StringType('hello'), celpy.celtypes.StringType('h')))"""), + celtypes.BoolType(True), + "_._(_)"), + ('"hello".endsWith("o")', + dedent("""\ + CEL = celpy.evaluation.result(base_activation, lambda activation: celpy.evaluation.function_endsWith(celpy.celtypes.StringType('hello'), celpy.celtypes.StringType('o')))"""), + celtypes.BoolType(True), + "_._(_)"), +] + +member_dot_arg_method_macro = [ + ('["hello", "world"].map(x, x) == ["hello", "world"]', + dedent("""\ + # member_dot_arg map: + ex_10_l = lambda activation: celpy.celtypes.ListType([celpy.celtypes.StringType('hello'), celpy.celtypes.StringType('world')]) + ex_10_x = lambda activation: activation.x + ex_10 = lambda activation: celpy.evaluation.macro_map(activation, 'x', ex_10_x, ex_10_l) + CEL = celpy.evaluation.result(base_activation, lambda activation: celpy.evaluation.bool_eq(ex_10(activation), celpy.celtypes.ListType([celpy.celtypes.StringType('hello'), celpy.celtypes.StringType('world')])))"""), + celtypes.BoolType(True), + "_._(_)"), + ('[true, false].filter(x, x) == [true]', + dedent("""\ + # member_dot_arg filter: + ex_10_l = lambda activation: celpy.celtypes.ListType([celpy.celtypes.BoolType(True), celpy.celtypes.BoolType(False)]) + ex_10_x = lambda activation: activation.x + ex_10 = lambda activation: celpy.evaluation.macro_filter(activation, 'x', ex_10_x, ex_10_l) + CEL = celpy.evaluation.result(base_activation, lambda activation: celpy.evaluation.bool_eq(ex_10(activation), celpy.celtypes.ListType([celpy.celtypes.BoolType(True)])))"""), + celtypes.BoolType(True), + "_._(_)"), + ('[42, 0].filter(x, 2 / x > 0) == [42]', + dedent("""\ + # member_dot_arg filter: + ex_10_l = lambda activation: celpy.celtypes.ListType([celpy.celtypes.IntType(42), celpy.celtypes.IntType(0)]) + ex_10_x = lambda activation: celpy.evaluation.bool_gt(operator.truediv(celpy.celtypes.IntType(2), activation.x), celpy.celtypes.IntType(0)) + ex_10 = lambda activation: celpy.evaluation.macro_filter(activation, 'x', ex_10_x, ex_10_l) + CEL = celpy.evaluation.result(base_activation, lambda activation: celpy.evaluation.bool_eq(ex_10(activation), celpy.celtypes.ListType([celpy.celtypes.IntType(42)])))"""), + CELEvalError, + "_._(_)"), + ('[true, false].exists_one(x, x)', + dedent("""\ + # member_dot_arg exists_one: + ex_8_l = lambda activation: celpy.celtypes.ListType([celpy.celtypes.BoolType(True), celpy.celtypes.BoolType(False)]) + ex_8_x = lambda activation: activation.x + ex_8 = lambda activation: celpy.evaluation.macro_exists_one(activation, 'x', ex_8_x, ex_8_l) + CEL = celpy.evaluation.result(base_activation, ex_8)"""), + celtypes.BoolType(True), + "_._(_)"), + ('[42, 0].exists_one(x, 2 / x > 0) == true', + dedent("""\ + # member_dot_arg exists_one: + ex_10_l = lambda activation: celpy.celtypes.ListType([celpy.celtypes.IntType(42), celpy.celtypes.IntType(0)]) + ex_10_x = lambda activation: celpy.evaluation.bool_gt(operator.truediv(celpy.celtypes.IntType(2), activation.x), celpy.celtypes.IntType(0)) + ex_10 = lambda activation: celpy.evaluation.macro_exists_one(activation, 'x', ex_10_x, ex_10_l) + CEL = celpy.evaluation.result(base_activation, lambda activation: celpy.evaluation.bool_eq(ex_10(activation), celpy.celtypes.BoolType(True)))"""), + CELEvalError, + "_._(_)"), + ('[true, false].exists(x, x)', + dedent("""\ + # member_dot_arg exists: + ex_8_l = lambda activation: celpy.celtypes.ListType([celpy.celtypes.BoolType(True), celpy.celtypes.BoolType(False)]) + ex_8_x = lambda activation: activation.x + ex_8 = lambda activation: celpy.evaluation.macro_exists(activation, 'x', ex_8_x, ex_8_l) + CEL = celpy.evaluation.result(base_activation, ex_8)"""), + celtypes.BoolType(True), + "_._(_)"), + ('[true, false].all(x, x)', + dedent("""\ + # member_dot_arg all: + ex_8_l = lambda activation: celpy.celtypes.ListType([celpy.celtypes.BoolType(True), celpy.celtypes.BoolType(False)]) + ex_8_x = lambda activation: activation.x + ex_8 = lambda activation: celpy.evaluation.macro_all(activation, 'x', ex_8_x, ex_8_l) + CEL = celpy.evaluation.result(base_activation, ex_8)"""), + celtypes.BoolType(False), + "_._(_)"), + + # Some difficult cases from the acceptance test suite, repeated here to make debugging easier. + ("[1, 'foo', 3].exists(e, e != '1')", + dedent("""\ + # member_dot_arg exists: + ex_8_l = lambda activation: celpy.celtypes.ListType([celpy.celtypes.IntType(1), celpy.celtypes.StringType('foo'), celpy.celtypes.IntType(3)]) + ex_8_x = lambda activation: celpy.evaluation.bool_ne(activation.e, celpy.celtypes.StringType('1')) + ex_8 = lambda activation: celpy.evaluation.macro_exists(activation, 'e', ex_8_x, ex_8_l) + CEL = celpy.evaluation.result(base_activation, ex_8) + """), + celtypes.BoolType(True), + "_._(_)"), + + ("['foal', 'foo', 'four'].exists_one(n, n.startsWith('fo'))", + dedent(""" + # member_dot_arg exists_one: + ex_8_l = lambda activation: celpy.celtypes.ListType([celpy.celtypes.StringType('foal'), celpy.celtypes.StringType('foo'), celpy.celtypes.StringType('four')]) + ex_8_x = lambda activation: celpy.evaluation.function_startsWith(activation.n, celpy.celtypes.StringType('fo')) + ex_8 = lambda activation: celpy.evaluation.macro_exists_one(activation, 'n', ex_8_x, ex_8_l) + CEL = celpy.evaluation.result(base_activation, ex_8)"""), + celtypes.BoolType(False), + "_._()"), +] + +# Requires variable `duration` amd type `protobuf_message` in activation. +dot_ident_arg = [ + ("duration.getMilliseconds()", + dedent("""\ + CEL = celpy.evaluation.result(base_activation, lambda activation: celpy.evaluation.function_getMilliseconds(activation.duration))"""), + celtypes.IntType(123123), + '._(_)'), + ('.duration', "CEL = celpy.evaluation.result(base_activation, lambda activation: activation.resolve_variable('duration'))", celtypes.DurationType(seconds=123, nanos=123456789), '_.IDENT'), + ('.protobuf_message().field', + dedent("""\ + CEL = celpy.evaluation.result(base_activation, lambda activation: activation.resolve_variable('protobuf_message')().get('field'))"""), + celtypes.IntType(42), + '_.IDENT()'), + ('.protobuf_message({"field": 42}).field', + dedent("""\ + CEL = celpy.evaluation.result(base_activation, lambda activation: activation.resolve_variable('protobuf_message')(celpy.celtypes.MapType([(celpy.celtypes.StringType('field'), celpy.celtypes.IntType(42))])).get('field'))"""), + celtypes.IntType(42), + '_.IDENT(_)'), + ('no_arg_function()', + "CEL = celpy.evaluation.result(base_activation, lambda activation: test_transpilation.no_arg_function())", + celtypes.IntType(42), + 'IDENT()'), +] + +unbound_names = [ + ("unknown_function(42)", + dedent("""CEL = celpy.evaluation.result(base_activation, lambda activation: CELEvalError('unbound function', KeyError, ('unknown_function',))(celpy.celtypes.IntType(42)))"""), + CELEvalError, + 'IDENT()' + ) +] + + +@pytest.fixture( + params=sum( + [ + literals, + function_params, method_params, macro_has_params, + unary_operator_params, binary_operator_params, short_circuit_params, + member_dot_params, member_item_params, member_object_params, + member_dot_arg_method, member_dot_arg_method_macro, + dot_ident_arg, unbound_names, + ], + [] + ), + ids=lambda f: f"{f[3]} Using {f[0]!r} ==> {f[2]}") +def binop_example(request): + source, expected, value, function_name = request.param + return source, expected, value, function_name + +@pytest.fixture(scope="module") +def transpiling_parser(): + # Reset the ClassVar CEL_PARSER. + celpy.CELParser.CEL_PARSER = None + parser = celpy.CELParser(tree_class=celpy.evaluation.TranspilerTree) + return parser + +def test_core_transpile(binop_example, mock_protobuf, mock_activation, mock_functions, transpiling_parser): + source, expected, expected_value, function_name = binop_example + tree = transpiling_parser.parse(source) + + tp = Transpiler(ast=tree, activation=mock_activation) # , functions=mock_functions) + tp.transpile() + print(tp.source_text.strip()) + assert tp.source_text.strip() == expected.strip() + + if isinstance(expected_value, type) and issubclass(expected_value, Exception): + with pytest.raises(expected_value): + computed = tp.evaluate({}) + else: + computed = tp.evaluate({}) + if expected_value is sentinel.MESSAGE: + assert computed == mock_protobuf.return_value + else: + assert computed == expected_value diff --git a/textproto/basic.feature b/textproto/basic.feature new file mode 100644 index 0000000..cc2f939 --- /dev/null +++ b/textproto/basic.feature @@ -0,0 +1,323 @@ + +Feature: basic + Basic conformance tests that all implementations should pass. + +# self_eval_zeroish -- Simple self-evaluating forms to zero-ish values. + +Scenario: self_eval_int_zero + + When CEL expression "0" is evaluated + # int64_value:0 + Then value is celpy.celtypes.IntType(source=0) + + +Scenario: self_eval_uint_zero + + When CEL expression "0u" is evaluated + # uint64_value:0 + Then value is celpy.celtypes.UintType(source=0) + + +Scenario: self_eval_float_zero + + When CEL expression "0.0" is evaluated + # double_value:0 + Then value is celpy.celtypes.DoubleType(source=0) + + +Scenario: self_eval_float_zerowithexp + + When CEL expression "0e+0" is evaluated + # double_value:0 + Then value is celpy.celtypes.DoubleType(source=0) + + +Scenario: self_eval_string_empty + + When CEL expression "''" is evaluated + # string_value:"" + Then value is celpy.celtypes.StringType(source='') + + +Scenario: self_eval_string_empty_quotes + + When CEL expression '""' is evaluated + # string_value:"" + Then value is celpy.celtypes.StringType(source='') + + +Scenario: self_eval_string_raw_prefix + + When CEL expression 'r""' is evaluated + # string_value:"" + Then value is celpy.celtypes.StringType(source='') + + +Scenario: self_eval_bytes_empty + + When CEL expression 'b""' is evaluated + # bytes_value:"" + Then value is celpy.celtypes.BytesType(source=b'') + + +Scenario: self_eval_bool_false + + When CEL expression "false" is evaluated + # bool_value:false + Then value is celpy.celtypes.BoolType(source=False) + + +Scenario: self_eval_null + + When CEL expression "null" is evaluated + # null_value:NULL_VALUE + Then value is None + + +Scenario: self_eval_empty_list + + When CEL expression "[]" is evaluated + # list_value:{} + Then value is [] + + +Scenario: self_eval_empty_map + + When CEL expression "{}" is evaluated + # map_value:{} + Then value is MapType({}) + + +Scenario: self_eval_string_raw_prefix_triple_double + + When CEL expression 'r""""""' is evaluated + # string_value:"" + Then value is celpy.celtypes.StringType(source='') + + +Scenario: self_eval_string_raw_prefix_triple_single + + When CEL expression "r''''''" is evaluated + # string_value:"" + Then value is celpy.celtypes.StringType(source='') + + + +# self_eval_nonzeroish -- Simple self-evaluating forms to non-zero-ish values. + +Scenario: self_eval_int_nonzero + + When CEL expression "42" is evaluated + # int64_value:42 + Then value is celpy.celtypes.IntType(source=42) + + +Scenario: self_eval_uint_nonzero + + When CEL expression "123456789u" is evaluated + # uint64_value:123456789 + Then value is celpy.celtypes.UintType(source=123456789) + + +Scenario: self_eval_int_negative_min + + When CEL expression "-9223372036854775808" is evaluated + # int64_value:-9223372036854775808 + Then value is celpy.celtypes.IntType(source=-9223372036854775808) + + +Scenario: self_eval_float_negative_exp + + When CEL expression "-2.3e+1" is evaluated + # double_value:-23 + Then value is celpy.celtypes.DoubleType(source=-23) + + +Scenario: self_eval_string_excl + + When CEL expression '"!"' is evaluated + # string_value:"!" + Then value is celpy.celtypes.StringType(source='!') + + +Scenario: self_eval_string_escape + + When CEL expression "'\''" is evaluated + # string_value:"'" + Then value is celpy.celtypes.StringType(source="'") + + +Scenario: self_eval_bytes_escape + + When CEL expression "b'ÿ'" is evaluated + # bytes_value:"ÿ" + Then value is celpy.celtypes.BytesType(source=b'\xc3\xbf') + + +Scenario: self_eval_bytes_invalid_utf8 + + When CEL expression "b'\000\xff'" is evaluated + # bytes_value:"\x00\xff" + Then value is celpy.celtypes.BytesType(source=b'\x00\xff') + + +Scenario: self_eval_list_singleitem + + When CEL expression "[-1]" is evaluated + # list_value:{values:{int64_value:-1}} + Then value is [celpy.celtypes.IntType(source=-1)] + + +Scenario: self_eval_map_singleitem + + When CEL expression '{"k":"v"}' is evaluated + # map_value:{entries:{key:{string_value:"k"} value:{string_value:"v"}}} + Then value is MapType({celpy.celtypes.StringType(source='k'): celpy.celtypes.StringType(source='v')}) + + +Scenario: self_eval_bool_true + + When CEL expression "true" is evaluated + # bool_value:true + Then value is celpy.celtypes.BoolType(source=True) + + +Scenario: self_eval_int_hex + + When CEL expression "0x55555555" is evaluated + # int64_value:1431655765 + Then value is celpy.celtypes.IntType(source=1431655765) + + +Scenario: self_eval_int_hex_negative + + When CEL expression "-0x55555555" is evaluated + # int64_value:-1431655765 + Then value is celpy.celtypes.IntType(source=-1431655765) + + +Scenario: self_eval_uint_hex + + When CEL expression "0x55555555u" is evaluated + # uint64_value:1431655765 + Then value is celpy.celtypes.UintType(source=1431655765) + + +Scenario: self_eval_unicode_escape_four + + When CEL expression '"\u270c"' is evaluated + # string_value:"✌" + Then value is celpy.celtypes.StringType(source='\u270c') + + +Scenario: self_eval_unicode_escape_eight + + When CEL expression '"\U0001f431"' is evaluated + # string_value:"🐱" + Then value is celpy.celtypes.StringType(source='\U0001f431') + + +Scenario: self_eval_ascii_escape_seq + + When CEL expression '"\a\b\f\n\r\t\v\"\'\\"' is evaluated + # string_value:"\x07\x08\x0c\n\r\t\x0b\"'\\" + Then value is celpy.celtypes.StringType(source='\x07\x08\x0c\n\r\t\x0b"\'\\') + + + +# variables -- Variable lookups. + +Scenario: self_eval_bound_lookup + + # type:{primitive:INT64} + Given type_env parameter "x" is celpy.celtypes.IntType + + # int64_value:123 + Given bindings parameter "x" is celpy.celtypes.IntType(source=123) + + When CEL expression "x" is evaluated + # int64_value:123 + Then value is celpy.celtypes.IntType(source=123) + + +Scenario: self_eval_unbound_lookup + An unbound variable should be marked as an error during execution. See google/cel-go#154 + When CEL expression "x" is evaluated + # errors:{message:"undeclared reference to 'x' (in container '')"} + Then eval_error is "undeclared reference to 'x' (in container '')" + + +Scenario: unbound_is_runtime_error + Make sure we can short-circuit around an unbound variable. + When CEL expression "x || true" is evaluated + # bool_value:true + Then value is celpy.celtypes.BoolType(source=True) + + + +# functions -- Basic mechanisms for function calls. + +Scenario: binop + + When CEL expression "1 + 1" is evaluated + # int64_value:2 + Then value is celpy.celtypes.IntType(source=2) + + +Scenario: unbound + + When CEL expression "f_unknown(17)" is evaluated + # errors:{message:"unbound function"} + Then eval_error is 'unbound function' + + +Scenario: unbound_is_runtime_error + + When CEL expression "f_unknown(17) || true" is evaluated + # bool_value:true + Then value is celpy.celtypes.BoolType(source=True) + + + +# reserved_const -- Named constants should never be shadowed by identifiers. + +Scenario: false + + # type:{primitive:BOOL} + Given type_env parameter "false" is celpy.celtypes.BoolType + + # bool_value:true + Given bindings parameter "false" is celpy.celtypes.BoolType(source=True) + + When CEL expression "false" is evaluated + # bool_value:false + Then value is celpy.celtypes.BoolType(source=False) + + +Scenario: true + + # type:{primitive:BOOL} + Given type_env parameter "true" is celpy.celtypes.BoolType + + # bool_value:false + Given bindings parameter "true" is celpy.celtypes.BoolType(source=False) + + When CEL expression "true" is evaluated + # bool_value:true + Then value is celpy.celtypes.BoolType(source=True) + + +Scenario: null + + # type:{primitive:BOOL} + Given type_env parameter "null" is celpy.celtypes.BoolType + + # bool_value:true + Given bindings parameter "null" is celpy.celtypes.BoolType(source=True) + + When CEL expression "null" is evaluated + # null_value:NULL_VALUE + Then value is None + + + diff --git a/textproto/basic.interim b/textproto/basic.interim new file mode 100644 index 0000000..6ffd7c1 --- /dev/null +++ b/textproto/basic.interim @@ -0,0 +1,275 @@ + +Feature: basic + Basic conformance tests that all implementations should pass. + +# self_eval_zeroish -- Simple self-evaluating forms to zero-ish values. + +Scenario: self_eval_int_zero + + When CEL expression "0" is evaluated + Then value is int64_value:0 + + +Scenario: self_eval_uint_zero + + When CEL expression "0u" is evaluated + Then value is uint64_value:0 + + +Scenario: self_eval_float_zero + + When CEL expression "0.0" is evaluated + Then value is double_value:0 + + +Scenario: self_eval_float_zerowithexp + + When CEL expression "0e+0" is evaluated + Then value is double_value:0 + + +Scenario: self_eval_string_empty + + When CEL expression "''" is evaluated + Then value is string_value:"" + + +Scenario: self_eval_string_empty_quotes + + When CEL expression "\"\"" is evaluated + Then value is string_value:"" + + +Scenario: self_eval_string_raw_prefix + + When CEL expression "r\"\"" is evaluated + Then value is string_value:"" + + +Scenario: self_eval_bytes_empty + + When CEL expression "b\"\"" is evaluated + Then value is bytes_value:"" + + +Scenario: self_eval_bool_false + + When CEL expression "false" is evaluated + Then value is bool_value:false + + +Scenario: self_eval_null + + When CEL expression "null" is evaluated + Then value is null_value:NULL_VALUE + + +Scenario: self_eval_empty_list + + When CEL expression "[]" is evaluated + Then value is list_value:{} + + +Scenario: self_eval_empty_map + + When CEL expression "{}" is evaluated + Then value is map_value:{} + + +Scenario: self_eval_string_raw_prefix_triple_double + + When CEL expression "r\"\"\"\"\"\"" is evaluated + Then value is string_value:"" + + +Scenario: self_eval_string_raw_prefix_triple_single + + When CEL expression "r''''''" is evaluated + Then value is string_value:"" + + + +# self_eval_nonzeroish -- Simple self-evaluating forms to non-zero-ish values. + +Scenario: self_eval_int_nonzero + + When CEL expression "42" is evaluated + Then value is int64_value:42 + + +Scenario: self_eval_uint_nonzero + + When CEL expression "123456789u" is evaluated + Then value is uint64_value:123456789 + + +Scenario: self_eval_int_negative_min + + When CEL expression "-9223372036854775808" is evaluated + Then value is int64_value:-9223372036854775808 + + +Scenario: self_eval_float_negative_exp + + When CEL expression "-2.3e+1" is evaluated + Then value is double_value:-23 + + +Scenario: self_eval_string_excl + + When CEL expression "\"!\"" is evaluated + Then value is string_value:"!" + + +Scenario: self_eval_string_escape + + When CEL expression "'\\''" is evaluated + Then value is string_value:"'" + + +Scenario: self_eval_bytes_escape + + When CEL expression "b'ÿ'" is evaluated + Then value is bytes_value:"ÿ" + + +Scenario: self_eval_bytes_invalid_utf8 + + When CEL expression "b'\\000\\xff'" is evaluated + Then value is bytes_value:"\x00\xff" + + +Scenario: self_eval_list_singleitem + + When CEL expression "[-1]" is evaluated + Then value is list_value:{values:{int64_value:-1}} + + +Scenario: self_eval_map_singleitem + + When CEL expression "{\"k\":\"v\"}" is evaluated + Then value is map_value:{entries:{key:{string_value:"k"} value:{string_value:"v"}}} + + +Scenario: self_eval_bool_true + + When CEL expression "true" is evaluated + Then value is bool_value:true + + +Scenario: self_eval_int_hex + + When CEL expression "0x55555555" is evaluated + Then value is int64_value:1431655765 + + +Scenario: self_eval_int_hex_negative + + When CEL expression "-0x55555555" is evaluated + Then value is int64_value:-1431655765 + + +Scenario: self_eval_uint_hex + + When CEL expression "0x55555555u" is evaluated + Then value is uint64_value:1431655765 + + +Scenario: self_eval_unicode_escape_four + + When CEL expression "\"\\u270c\"" is evaluated + Then value is string_value:"✌" + + +Scenario: self_eval_unicode_escape_eight + + When CEL expression "\"\\U0001f431\"" is evaluated + Then value is string_value:"🐱" + + +Scenario: self_eval_ascii_escape_seq + + When CEL expression "\"\\a\\b\\f\\n\\r\\t\\v\\\"\\'\\\\\"" is evaluated + Then value is string_value:"\x07\x08\x0c\n\r\t\x0b\"'\\" + + + +# variables -- Variable lookups. + +Scenario: self_eval_bound_lookup + + Given type_env parameter "x" is &{type:{primitive:INT64}} + + Given bindings parameter "x" is int64_value:123 + + When CEL expression "x" is evaluated + Then value is int64_value:123 + + +Scenario: self_eval_unbound_lookup + An unbound variable should be marked as an error during execution. See google/cel-go#154 + When CEL expression "x" is evaluated + Then eval_error is errors:{message:"undeclared reference to 'x' (in container '')"} + + +Scenario: unbound_is_runtime_error + Make sure we can short-circuit around an unbound variable. + When CEL expression "x || true" is evaluated + Then value is bool_value:true + + + +# functions -- Basic mechanisms for function calls. + +Scenario: binop + + When CEL expression "1 + 1" is evaluated + Then value is int64_value:2 + + +Scenario: unbound + + When CEL expression "f_unknown(17)" is evaluated + Then eval_error is errors:{message:"unbound function"} + + +Scenario: unbound_is_runtime_error + + When CEL expression "f_unknown(17) || true" is evaluated + Then value is bool_value:true + + + +# reserved_const -- Named constants should never be shadowed by identifiers. + +Scenario: false + + Given type_env parameter "false" is &{type:{primitive:BOOL}} + + Given bindings parameter "false" is bool_value:true + + When CEL expression "false" is evaluated + Then value is bool_value:false + + +Scenario: true + + Given type_env parameter "true" is &{type:{primitive:BOOL}} + + Given bindings parameter "true" is bool_value:false + + When CEL expression "true" is evaluated + Then value is bool_value:true + + +Scenario: null + + Given type_env parameter "null" is &{type:{primitive:BOOL}} + + Given bindings parameter "null" is bool_value:true + + When CEL expression "null" is evaluated + Then value is null_value:NULL_VALUE + + + diff --git a/textproto/basic.textproto b/textproto/basic.textproto new file mode 100644 index 0000000..81fd98d --- /dev/null +++ b/textproto/basic.textproto @@ -0,0 +1,275 @@ +name: "basic" +description: "Basic conformance tests that all implementations should pass." +section { + name: "self_eval_zeroish" + description: "Simple self-evaluating forms to zero-ish values." + test { + name: "self_eval_int_zero" + expr: "0" + value: { int64_value: 0 } + } + test { + name: "self_eval_uint_zero" + expr: "0u" + value: { uint64_value: 0 } + } + test { + name: "self_eval_float_zero" + expr: "0.0" + value: { double_value: 0 } + } + test { + name: "self_eval_float_zerowithexp" + expr: "0e+0" + value: { double_value: 0 } + } + test { + name: "self_eval_string_empty" + expr: "''" + value: { string_value: "" } + } + test { + name: "self_eval_string_empty_quotes" + expr: '""' + value: { string_value: "" } + } + test { + name: "self_eval_string_raw_prefix" + expr: 'r""' + value: { string_value: "" } + } + test { + name: "self_eval_bytes_empty" + expr: 'b""' + value: { bytes_value: "" } + } + test { + name: "self_eval_bool_false" + expr: "false" + value: { bool_value: false } + } + test { + name: "self_eval_null" + expr: "null" + value: { null_value: NULL_VALUE } + } + test { + name: "self_eval_empty_list" + expr: '[]' + value: { list_value: {} } + } + test { + name: "self_eval_empty_map" + expr: '{}' + value: { map_value: {} } + } + test { + name: "self_eval_string_raw_prefix_triple_double" + expr: 'r""""""' + value: { string_value: "" } + } + test { + name: "self_eval_string_raw_prefix_triple_single" + expr: "r''''''" + value: { string_value: "" } + } +} +section { + name: "self_eval_nonzeroish" + description: "Simple self-evaluating forms to non-zero-ish values." + test { + name: "self_eval_int_nonzero" + expr: "42" + value: { int64_value: 42 } + } + test { + name: "self_eval_uint_nonzero" + expr: "123456789u" + value: { uint64_value: 123456789 } + } + test { + name: "self_eval_int_negative_min" + expr: "-9223372036854775808" + value: { int64_value: -9223372036854775808 } + } + test { + name: "self_eval_float_negative_exp" + expr: "-2.3e+1" + value: { double_value: -23.0 } + } + test { + name: "self_eval_string_excl" + expr: '"!"' + value: { string_value: "!" } + } + test { + name: "self_eval_string_escape" + expr: "'\\''" + value: { string_value: "'" } + } + test { + name: "self_eval_bytes_escape" + expr: "b'ÿ'" + value: { bytes_value: "\303\277" } + } + test { + name: "self_eval_bytes_invalid_utf8" + expr: "b'\\000\\xff'" + value: { bytes_value: "\000\377" } + } + test { + name: "self_eval_list_singleitem" + expr: "[-1]" + value: { + list_value { + values: { int64_value: -1 } + } + } + } + test { + name: "self_eval_map_singleitem" + expr: '{"k":"v"}' + value: { + map_value { + entries { + key: { string_value: "k" } + value: { string_value: "v" } + } + } + } + } + test { + name: "self_eval_bool_true" + expr: "true" + value: { bool_value: true } + } + test { + name: "self_eval_int_hex" + expr: "0x55555555" + value: { int64_value: 1431655765 } + } + test { + name: "self_eval_int_hex_negative" + expr: "-0x55555555" + value: { int64_value: -1431655765 } + } + test { + name: "self_eval_uint_hex" + expr: "0x55555555u" + value: { uint64_value: 1431655765 } + } + test { + name: "self_eval_unicode_escape_four" + expr: '"\\u270c"' + value: { string_value: "\xe2\x9c\x8c" } + } + test { + name: "self_eval_unicode_escape_eight" + expr: '"\\U0001f431"' + value: { string_value: "\xf0\x9f\x90\xb1" } + } + test { + name: "self_eval_ascii_escape_seq" + expr: '"\\a\\b\\f\\n\\r\\t\\v\\"\\\'\\\\"' + value: { string_value: "\a\b\f\n\r\t\v\"'\\" } + } +} +section { + name: "variables" + description: "Variable lookups." + test { + name: "self_eval_bound_lookup" + expr: "x" + type_env: { + name: "x", + ident: { type: { primitive: INT64 } } + } + bindings: { + key: "x" + value: { value: { int64_value: 123 } } + } + value: { int64_value: 123 } + } + test { + name: "self_eval_unbound_lookup" + description: "An unbound variable should be marked as an error during execution. See google/cel-go#154" + expr: "x" + disable_check: true + eval_error: { + errors: { message: "undeclared reference to 'x' (in container '')" } + } + } + test { + name: "unbound_is_runtime_error" + description: "Make sure we can short-circuit around an unbound variable." + expr: "x || true" + disable_check: true + value { bool_value: true } + } +} +section { + name: "functions" + description: "Basic mechanisms for function calls." + test { + name: "binop" + expr: "1 + 1" + value { int64_value: 2 } + } + test { + name: "unbound" + expr: "f_unknown(17)" + disable_check: true + eval_error { + errors { message: "unbound function" } + } + } + test { + name: "unbound_is_runtime_error" + expr: "f_unknown(17) || true" + disable_check: true + value { bool_value: true } + } +} +section { + name: "reserved_const" + description: "Named constants should never be shadowed by identifiers." + test { + name: "false" + expr: "false" + type_env: { + name: "false" + ident: { type: { primitive: BOOL } } + } + bindings { + key: "false" + value: { value: { bool_value: true } } + } + value: { bool_value: false } + } + test { + name: "true" + expr: "true" + type_env: { + name: "true" + ident: { type: { primitive: BOOL } } + } + bindings { + key: "true" + value: { value: { bool_value: false } } + } + value: { bool_value: true } + } + test { + name: "null" + expr: "null" + type_env: { + name: "null" + ident: { type: { primitive: BOOL } } + } + bindings { + key: "null" + value: { value: { bool_value: true } } + } + value: { null_value: 0 } + } +} diff --git a/textproto/conversions.feature b/textproto/conversions.feature new file mode 100644 index 0000000..697dbb6 --- /dev/null +++ b/textproto/conversions.feature @@ -0,0 +1,571 @@ + +Feature: conversions + Tests for type conversions. + +# bytes -- Conversions to bytes. + +Scenario: string_empty + + When CEL expression "bytes('')" is evaluated + # bytes_value:"" + Then value is celpy.celtypes.BytesType(source=b'') + + +Scenario: string + + When CEL expression "bytes('abc')" is evaluated + # bytes_value:"abc" + Then value is celpy.celtypes.BytesType(source=b'abc') + + +Scenario: string_unicode + + When CEL expression "bytes('ÿ')" is evaluated + # bytes_value:"ÿ" + Then value is celpy.celtypes.BytesType(source=b'\xc3\xbf') + + +Scenario: string_unicode_vs_literal + + When CEL expression "bytes('\377') == b'\377'" is evaluated + # bool_value:false + Then value is celpy.celtypes.BoolType(source=False) + + + +# double -- Conversions to double. + +Scenario: int_zero + + When CEL expression "double(0)" is evaluated + # double_value:0 + Then value is celpy.celtypes.DoubleType(source=0) + + +Scenario: int_pos + + When CEL expression "double(1000000000000)" is evaluated + # double_value:1e+12 + Then value is celpy.celtypes.DoubleType(source=1000000000000.0) + + +Scenario: int_neg + + When CEL expression "double(-1000000000000000)" is evaluated + # double_value:-1e+15 + Then value is celpy.celtypes.DoubleType(source=-1000000000000000.0) + + +Scenario: int_range + Largest signed 64-bit. Rounds to nearest double. + When CEL expression "double(9223372036854775807)" is evaluated + # double_value:9.223372036854776e+18 + Then value is celpy.celtypes.DoubleType(source=9.223372036854776e+18) + + +Scenario: uint_zero + + When CEL expression "double(0u)" is evaluated + # double_value:0 + Then value is celpy.celtypes.DoubleType(source=0) + + +Scenario: uint_pos + + When CEL expression "double(123u)" is evaluated + # double_value:123 + Then value is celpy.celtypes.DoubleType(source=123) + + +Scenario: uint_range + Largest unsigned 64-bit. + When CEL expression "double(18446744073709551615u)" is evaluated + # double_value:1.8446744073709552e+19 + Then value is celpy.celtypes.DoubleType(source=1.8446744073709552e+19) + + +Scenario: string_zero + + When CEL expression "double('0')" is evaluated + # double_value:0 + Then value is celpy.celtypes.DoubleType(source=0) + + +Scenario: string_zero_dec + + When CEL expression "double('0.0')" is evaluated + # double_value:0 + Then value is celpy.celtypes.DoubleType(source=0) + + +Scenario: string_neg_zero + + When CEL expression "double('-0.0')" is evaluated + # double_value:-0 + Then value is celpy.celtypes.DoubleType(source=0) + + +Scenario: string_no_dec + + When CEL expression "double('123')" is evaluated + # double_value:123 + Then value is celpy.celtypes.DoubleType(source=123) + + +Scenario: string_pos + + When CEL expression "double('123.456')" is evaluated + # double_value:123.456 + Then value is celpy.celtypes.DoubleType(source=123.456) + + +Scenario: string_neg + + When CEL expression "double('-987.654')" is evaluated + # double_value:-987.654 + Then value is celpy.celtypes.DoubleType(source=-987.654) + + +Scenario: string_exp_pos_pos + + When CEL expression "double('6.02214e23')" is evaluated + # double_value:6.02214e+23 + Then value is celpy.celtypes.DoubleType(source=6.02214e+23) + + +Scenario: string_exp_pos_neg + + When CEL expression "double('1.38e-23')" is evaluated + # double_value:1.38e-23 + Then value is celpy.celtypes.DoubleType(source=1.38e-23) + + +Scenario: string_exp_neg_pos + + When CEL expression "double('-84.32e7')" is evaluated + # double_value:-8.432e+08 + Then value is celpy.celtypes.DoubleType(source=-843200000.0) + + +Scenario: string_exp_neg_neg + + When CEL expression "double('-5.43e-21')" is evaluated + # double_value:-5.43e-21 + Then value is celpy.celtypes.DoubleType(source=-5.43e-21) + + + +# dyn -- Tests for dyn annotation. + +Scenario: dyn_heterogeneous_list + No need to disable type checking. + When CEL expression "type(dyn([1, 'one']))" is evaluated + # type_value:"list" + Then value is celpy.celtypes.ListType + + + +# int -- Conversions to int. + +Scenario: uint + + When CEL expression "int(42u)" is evaluated + # int64_value:42 + Then value is celpy.celtypes.IntType(source=42) + + +Scenario: uint_zero + + When CEL expression "int(0u)" is evaluated + # int64_value:0 + Then value is celpy.celtypes.IntType(source=0) + + +Scenario: uint_range + + When CEL expression "int(18446744073709551615u)" is evaluated + # errors:{message:"range error"} + Then eval_error is 'range error' + + +Scenario: double_round_neg + + When CEL expression "int(-123.456)" is evaluated + # int64_value:-123 + Then value is celpy.celtypes.IntType(source=-123) + + +Scenario: double_nearest + + When CEL expression "int(1.9)" is evaluated + # int64_value:2 + Then value is celpy.celtypes.IntType(source=2) + + +Scenario: double_nearest_neg + + When CEL expression "int(-7.9)" is evaluated + # int64_value:-8 + Then value is celpy.celtypes.IntType(source=-8) + + +Scenario: double_half_away_pos + + When CEL expression "int(11.5)" is evaluated + # int64_value:12 + Then value is celpy.celtypes.IntType(source=12) + + +Scenario: double_half_away_neg + + When CEL expression "int(-3.5)" is evaluated + # int64_value:-4 + Then value is celpy.celtypes.IntType(source=-4) + + +Scenario: double_range + + When CEL expression "int(1e99)" is evaluated + # errors:{message:"range"} + Then eval_error is 'range' + + +Scenario: string + + When CEL expression "int('987')" is evaluated + # int64_value:987 + Then value is celpy.celtypes.IntType(source=987) + + +Scenario: timestamp + + When CEL expression "int(timestamp('2004-09-16T23:59:59Z'))" is evaluated + # int64_value:1095379199 + Then value is celpy.celtypes.IntType(source=1095379199) + + + +# string -- Conversions to string. + +Scenario: int + + When CEL expression "string(123)" is evaluated + # string_value:"123" + Then value is celpy.celtypes.StringType(source='123') + + +Scenario: int_neg + + When CEL expression "string(-456)" is evaluated + # string_value:"-456" + Then value is celpy.celtypes.StringType(source='-456') + + +Scenario: uint + + When CEL expression "string(9876u)" is evaluated + # string_value:"9876" + Then value is celpy.celtypes.StringType(source='9876') + + +Scenario: double + + When CEL expression "string(123.456)" is evaluated + # string_value:"123.456" + Then value is celpy.celtypes.StringType(source='123.456') + + +Scenario: double_hard + + When CEL expression "string(-4.5e-3)" is evaluated + # string_value:"-0.0045" + Then value is celpy.celtypes.StringType(source='-0.0045') + + +Scenario: bytes + + When CEL expression "string(b'abc')" is evaluated + # string_value:"abc" + Then value is celpy.celtypes.StringType(source='abc') + + +Scenario: bytes_unicode + + When CEL expression "string(b'\303\277')" is evaluated + # string_value:"ÿ" + Then value is celpy.celtypes.StringType(source='\xff') + + +Scenario: bytes_invalid + + When CEL expression "string(b'\000\xff')" is evaluated + # errors:{message:"invalid UTF-8"} + Then eval_error is 'invalid UTF-8' + + + +# type -- Type reflection tests. + +Scenario: bool + + When CEL expression "type(true)" is evaluated + # type_value:"bool" + Then value is celpy.celtypes.BoolType + + +Scenario: bool_denotation + + When CEL expression "bool" is evaluated + # type_value:"bool" + Then value is celpy.celtypes.BoolType + + +Scenario: dyn_no_denotation + + When CEL expression "dyn" is evaluated + # errors:{message:"unknown varaible"} + Then eval_error is 'unknown varaible' + + +Scenario: int + + When CEL expression "type(0)" is evaluated + # type_value:"int" + Then value is celpy.celtypes.IntType + + +Scenario: int_denotation + + When CEL expression "int" is evaluated + # type_value:"int" + Then value is celpy.celtypes.IntType + + +Scenario: eq_same + + When CEL expression "type(true) == type(false)" is evaluated + # bool_value:true + Then value is celpy.celtypes.BoolType(source=True) + + +Scenario: uint + + When CEL expression "type(64u)" is evaluated + # type_value:"uint" + Then value is celpy.celtypes.UintType + + +Scenario: uint_denotation + + When CEL expression "uint" is evaluated + # type_value:"uint" + Then value is celpy.celtypes.UintType + + +Scenario: double + + When CEL expression "type(3.14)" is evaluated + # type_value:"double" + Then value is celpy.celtypes.DoubleType + + +Scenario: double_denotation + + When CEL expression "double" is evaluated + # type_value:"double" + Then value is celpy.celtypes.DoubleType + + +Scenario: null_type + + When CEL expression "type(null)" is evaluated + # type_value:"null_type" + Then value is NoneType + + +Scenario: null_type_denotation + + When CEL expression "null_type" is evaluated + # type_value:"null_type" + Then value is NoneType + + +Scenario: string + + When CEL expression "type('foo')" is evaluated + # type_value:"string" + Then value is celpy.celtypes.StringType + + +Scenario: string_denotation + + When CEL expression "string" is evaluated + # type_value:"string" + Then value is celpy.celtypes.StringType + + +Scenario: bytes + + When CEL expression "type(b'\xff')" is evaluated + # type_value:"bytes" + Then value is celpy.celtypes.BytesType + + +Scenario: bytes_denotation + + When CEL expression "bytes" is evaluated + # type_value:"bytes" + Then value is celpy.celtypes.BytesType + + +Scenario: list + + When CEL expression "type([1, 2, 3])" is evaluated + # type_value:"list" + Then value is celpy.celtypes.ListType + + +Scenario: list_denotation + + When CEL expression "list" is evaluated + # type_value:"list" + Then value is celpy.celtypes.ListType + + +Scenario: lists_monomorphic + + When CEL expression "type([1, 2, 3]) == type(['one', 'two', 'three'])" is evaluated + # bool_value:true + Then value is celpy.celtypes.BoolType(source=True) + + +Scenario: map + + When CEL expression "type({4: 16})" is evaluated + # type_value:"map" + Then value is celpy.celtypes.MapType + + +Scenario: map_denotation + + When CEL expression "map" is evaluated + # type_value:"map" + Then value is celpy.celtypes.MapType + + +Scenario: map_monomorphic + + When CEL expression "type({'one': 1}) == type({1: 'one'})" is evaluated + # bool_value:true + Then value is celpy.celtypes.BoolType(source=True) + + +Scenario: eq_diff + + When CEL expression "type(7) == type(7u)" is evaluated + # bool_value:false + Then value is celpy.celtypes.BoolType(source=False) + + +Scenario: neq_same + + When CEL expression "type(0.0) != type(-0.0)" is evaluated + # bool_value:false + Then value is celpy.celtypes.BoolType(source=False) + + +Scenario: neq_diff + + When CEL expression "type(0.0) != type(0)" is evaluated + # bool_value:true + Then value is celpy.celtypes.BoolType(source=True) + + +Scenario: meta + + When CEL expression "type(type(7)) == type(type(7u))" is evaluated + # bool_value:true + Then value is celpy.celtypes.BoolType(source=True) + + +Scenario: type + + When CEL expression "type(int)" is evaluated + # type_value:"type" + Then value is celpy.celtypes.TypeType + + +Scenario: type_denotation + + When CEL expression "type" is evaluated + # type_value:"type" + Then value is celpy.celtypes.TypeType + + +Scenario: type_type + + When CEL expression "type(type)" is evaluated + # type_value:"type" + Then value is celpy.celtypes.TypeType + + + +# uint -- Conversions to uint. + +Scenario: int + + When CEL expression "uint(1729)" is evaluated + # uint64_value:1729 + Then value is celpy.celtypes.UintType(source=1729) + + +Scenario: int_neg + + When CEL expression "uint(-1)" is evaluated + # errors:{message:"range"} + Then eval_error is 'range' + + +Scenario: double + + When CEL expression "uint(3.14159265)" is evaluated + # uint64_value:3 + Then value is celpy.celtypes.UintType(source=3) + + +Scenario: double_nearest_int + + When CEL expression "int(1.9)" is evaluated + # int64_value:2 + Then value is celpy.celtypes.IntType(source=2) + + +Scenario: double_nearest + + When CEL expression "uint(1.9)" is evaluated + # uint64_value:2 + Then value is celpy.celtypes.UintType(source=2) + + +Scenario: double_half_away + + When CEL expression "uint(25.5)" is evaluated + # uint64_value:26 + Then value is celpy.celtypes.UintType(source=26) + + +Scenario: double_range + + When CEL expression "uint(6.022e23)" is evaluated + # errors:{message:"range"} + Then eval_error is 'range' + + +Scenario: string + + When CEL expression "uint('300')" is evaluated + # uint64_value:300 + Then value is celpy.celtypes.UintType(source=300) + + + diff --git a/textproto/conversions.interim b/textproto/conversions.interim new file mode 100644 index 0000000..465577a --- /dev/null +++ b/textproto/conversions.interim @@ -0,0 +1,493 @@ + +Feature: conversions + Tests for type conversions. + +# bytes -- Conversions to bytes. + +Scenario: string_empty + + When CEL expression "bytes('')" is evaluated + Then value is bytes_value:"" + + +Scenario: string + + When CEL expression "bytes('abc')" is evaluated + Then value is bytes_value:"abc" + + +Scenario: string_unicode + + When CEL expression "bytes('ÿ')" is evaluated + Then value is bytes_value:"ÿ" + + +Scenario: string_unicode_vs_literal + + When CEL expression "bytes('\\377') == b'\\377'" is evaluated + Then value is bool_value:false + + + +# double -- Conversions to double. + +Scenario: int_zero + + When CEL expression "double(0)" is evaluated + Then value is double_value:0 + + +Scenario: int_pos + + When CEL expression "double(1000000000000)" is evaluated + Then value is double_value:1e+12 + + +Scenario: int_neg + + When CEL expression "double(-1000000000000000)" is evaluated + Then value is double_value:-1e+15 + + +Scenario: int_range + Largest signed 64-bit. Rounds to nearest double. + When CEL expression "double(9223372036854775807)" is evaluated + Then value is double_value:9.223372036854776e+18 + + +Scenario: uint_zero + + When CEL expression "double(0u)" is evaluated + Then value is double_value:0 + + +Scenario: uint_pos + + When CEL expression "double(123u)" is evaluated + Then value is double_value:123 + + +Scenario: uint_range + Largest unsigned 64-bit. + When CEL expression "double(18446744073709551615u)" is evaluated + Then value is double_value:1.8446744073709552e+19 + + +Scenario: string_zero + + When CEL expression "double('0')" is evaluated + Then value is double_value:0 + + +Scenario: string_zero_dec + + When CEL expression "double('0.0')" is evaluated + Then value is double_value:0 + + +Scenario: string_neg_zero + + When CEL expression "double('-0.0')" is evaluated + Then value is double_value:-0 + + +Scenario: string_no_dec + + When CEL expression "double('123')" is evaluated + Then value is double_value:123 + + +Scenario: string_pos + + When CEL expression "double('123.456')" is evaluated + Then value is double_value:123.456 + + +Scenario: string_neg + + When CEL expression "double('-987.654')" is evaluated + Then value is double_value:-987.654 + + +Scenario: string_exp_pos_pos + + When CEL expression "double('6.02214e23')" is evaluated + Then value is double_value:6.02214e+23 + + +Scenario: string_exp_pos_neg + + When CEL expression "double('1.38e-23')" is evaluated + Then value is double_value:1.38e-23 + + +Scenario: string_exp_neg_pos + + When CEL expression "double('-84.32e7')" is evaluated + Then value is double_value:-8.432e+08 + + +Scenario: string_exp_neg_neg + + When CEL expression "double('-5.43e-21')" is evaluated + Then value is double_value:-5.43e-21 + + + +# dyn -- Tests for dyn annotation. + +Scenario: dyn_heterogeneous_list + No need to disable type checking. + When CEL expression "type(dyn([1, 'one']))" is evaluated + Then value is type_value:"list" + + + +# int -- Conversions to int. + +Scenario: uint + + When CEL expression "int(42u)" is evaluated + Then value is int64_value:42 + + +Scenario: uint_zero + + When CEL expression "int(0u)" is evaluated + Then value is int64_value:0 + + +Scenario: uint_range + + When CEL expression "int(18446744073709551615u)" is evaluated + Then eval_error is errors:{message:"range error"} + + +Scenario: double_round_neg + + When CEL expression "int(-123.456)" is evaluated + Then value is int64_value:-123 + + +Scenario: double_nearest + + When CEL expression "int(1.9)" is evaluated + Then value is int64_value:2 + + +Scenario: double_nearest_neg + + When CEL expression "int(-7.9)" is evaluated + Then value is int64_value:-8 + + +Scenario: double_half_away_pos + + When CEL expression "int(11.5)" is evaluated + Then value is int64_value:12 + + +Scenario: double_half_away_neg + + When CEL expression "int(-3.5)" is evaluated + Then value is int64_value:-4 + + +Scenario: double_range + + When CEL expression "int(1e99)" is evaluated + Then eval_error is errors:{message:"range"} + + +Scenario: string + + When CEL expression "int('987')" is evaluated + Then value is int64_value:987 + + +Scenario: timestamp + + When CEL expression "int(timestamp('2004-09-16T23:59:59Z'))" is evaluated + Then value is int64_value:1095379199 + + + +# string -- Conversions to string. + +Scenario: int + + When CEL expression "string(123)" is evaluated + Then value is string_value:"123" + + +Scenario: int_neg + + When CEL expression "string(-456)" is evaluated + Then value is string_value:"-456" + + +Scenario: uint + + When CEL expression "string(9876u)" is evaluated + Then value is string_value:"9876" + + +Scenario: double + + When CEL expression "string(123.456)" is evaluated + Then value is string_value:"123.456" + + +Scenario: double_hard + + When CEL expression "string(-4.5e-3)" is evaluated + Then value is string_value:"-0.0045" + + +Scenario: bytes + + When CEL expression "string(b'abc')" is evaluated + Then value is string_value:"abc" + + +Scenario: bytes_unicode + + When CEL expression "string(b'\\303\\277')" is evaluated + Then value is string_value:"ÿ" + + +Scenario: bytes_invalid + + When CEL expression "string(b'\\000\\xff')" is evaluated + Then eval_error is errors:{message:"invalid UTF-8"} + + + +# type -- Type reflection tests. + +Scenario: bool + + When CEL expression "type(true)" is evaluated + Then value is type_value:"bool" + + +Scenario: bool_denotation + + When CEL expression "bool" is evaluated + Then value is type_value:"bool" + + +Scenario: dyn_no_denotation + + When CEL expression "dyn" is evaluated + Then eval_error is errors:{message:"unknown varaible"} + + +Scenario: int + + When CEL expression "type(0)" is evaluated + Then value is type_value:"int" + + +Scenario: int_denotation + + When CEL expression "int" is evaluated + Then value is type_value:"int" + + +Scenario: eq_same + + When CEL expression "type(true) == type(false)" is evaluated + Then value is bool_value:true + + +Scenario: uint + + When CEL expression "type(64u)" is evaluated + Then value is type_value:"uint" + + +Scenario: uint_denotation + + When CEL expression "uint" is evaluated + Then value is type_value:"uint" + + +Scenario: double + + When CEL expression "type(3.14)" is evaluated + Then value is type_value:"double" + + +Scenario: double_denotation + + When CEL expression "double" is evaluated + Then value is type_value:"double" + + +Scenario: null_type + + When CEL expression "type(null)" is evaluated + Then value is type_value:"null_type" + + +Scenario: null_type_denotation + + When CEL expression "null_type" is evaluated + Then value is type_value:"null_type" + + +Scenario: string + + When CEL expression "type('foo')" is evaluated + Then value is type_value:"string" + + +Scenario: string_denotation + + When CEL expression "string" is evaluated + Then value is type_value:"string" + + +Scenario: bytes + + When CEL expression "type(b'\\xff')" is evaluated + Then value is type_value:"bytes" + + +Scenario: bytes_denotation + + When CEL expression "bytes" is evaluated + Then value is type_value:"bytes" + + +Scenario: list + + When CEL expression "type([1, 2, 3])" is evaluated + Then value is type_value:"list" + + +Scenario: list_denotation + + When CEL expression "list" is evaluated + Then value is type_value:"list" + + +Scenario: lists_monomorphic + + When CEL expression "type([1, 2, 3]) == type(['one', 'two', 'three'])" is evaluated + Then value is bool_value:true + + +Scenario: map + + When CEL expression "type({4: 16})" is evaluated + Then value is type_value:"map" + + +Scenario: map_denotation + + When CEL expression "map" is evaluated + Then value is type_value:"map" + + +Scenario: map_monomorphic + + When CEL expression "type({'one': 1}) == type({1: 'one'})" is evaluated + Then value is bool_value:true + + +Scenario: eq_diff + + When CEL expression "type(7) == type(7u)" is evaluated + Then value is bool_value:false + + +Scenario: neq_same + + When CEL expression "type(0.0) != type(-0.0)" is evaluated + Then value is bool_value:false + + +Scenario: neq_diff + + When CEL expression "type(0.0) != type(0)" is evaluated + Then value is bool_value:true + + +Scenario: meta + + When CEL expression "type(type(7)) == type(type(7u))" is evaluated + Then value is bool_value:true + + +Scenario: type + + When CEL expression "type(int)" is evaluated + Then value is type_value:"type" + + +Scenario: type_denotation + + When CEL expression "type" is evaluated + Then value is type_value:"type" + + +Scenario: type_type + + When CEL expression "type(type)" is evaluated + Then value is type_value:"type" + + + +# uint -- Conversions to uint. + +Scenario: int + + When CEL expression "uint(1729)" is evaluated + Then value is uint64_value:1729 + + +Scenario: int_neg + + When CEL expression "uint(-1)" is evaluated + Then eval_error is errors:{message:"range"} + + +Scenario: double + + When CEL expression "uint(3.14159265)" is evaluated + Then value is uint64_value:3 + + +Scenario: double_nearest_int + + When CEL expression "int(1.9)" is evaluated + Then value is int64_value:2 + + +Scenario: double_nearest + + When CEL expression "uint(1.9)" is evaluated + Then value is uint64_value:2 + + +Scenario: double_half_away + + When CEL expression "uint(25.5)" is evaluated + Then value is uint64_value:26 + + +Scenario: double_range + + When CEL expression "uint(6.022e23)" is evaluated + Then eval_error is errors:{message:"range"} + + +Scenario: string + + When CEL expression "uint('300')" is evaluated + Then value is uint64_value:300 + + + diff --git a/textproto/conversions.textproto b/textproto/conversions.textproto new file mode 100644 index 0000000..6f46464 --- /dev/null +++ b/textproto/conversions.textproto @@ -0,0 +1,436 @@ +name: "conversions" +description: "Tests for type conversions." +section { + name: "bytes" + description: "Conversions to bytes." + test { + name: "string_empty" + expr: "bytes('')" + value: { bytes_value: "" } + } + test { + name: "string" + expr: "bytes('abc')" + value: { bytes_value: "abc" } + } + test { + name: "string_unicode" + expr: "bytes('ÿ')" + value: { bytes_value: "\303\277" } + } + test { + name: "string_unicode_vs_literal" + expr: "bytes('\\377') == b'\\377'" + value: { bool_value: false } + } +} +section { + name: "double" + description: "Conversions to double." + test { + name: "int_zero" + expr: "double(0)" + value: { double_value: 0.0 } + } + test { + name: "int_pos" + expr: "double(1000000000000)" + value: { double_value: 1e12 } + } + test { + name: "int_neg" + expr: "double(-1000000000000000)" + value: { double_value: -1e15 } + } + test { + name: "int_range" + description: "Largest signed 64-bit. Rounds to nearest double." + expr: "double(9223372036854775807)" + value: { double_value: 9.223372036854775807e18 } + } + test { + name: "uint_zero" + expr: "double(0u)" + value: { double_value: 0.0 } + } + test { + name: "uint_pos" + expr: "double(123u)" + value: { double_value: 123.0 } + } + test { + name: "uint_range" + description: "Largest unsigned 64-bit." + expr: "double(18446744073709551615u)" + value: { double_value: 1.8446744073709551615e19 } + } + test { + name: "string_zero" + expr: "double('0')" + value: { double_value: 0.0 } + } + test { + name: "string_zero_dec" + expr: "double('0.0')" + value: { double_value: 0.0 } + } + test { + name: "string_neg_zero" + expr: "double('-0.0')" + value: { double_value: -0.0 } + } + test { + name: "string_no_dec" + expr: "double('123')" + value: { double_value: 123.0 } + } + test { + name: "string_pos" + expr: "double('123.456')" + value: { double_value: 123.456 } + } + test { + name: "string_neg" + expr: "double('-987.654')" + value: { double_value: -987.654 } + } + test { + name: "string_exp_pos_pos" + expr: "double('6.02214e23')" + value: { double_value: 6.02214e23 } + } + test { + name: "string_exp_pos_neg" + expr: "double('1.38e-23')" + value: { double_value: 1.38e-23 } + } + test { + name: "string_exp_neg_pos" + expr: "double('-84.32e7')" + value: { double_value: -8.432e8 } + } + test { + name: "string_exp_neg_neg" + expr: "double('-5.43e-21')" + value: { double_value: -5.43e-21 } + } +} +section { + name: "dyn" + description: "Tests for dyn annotation." + test { + name: "dyn_heterogeneous_list" + description: "No need to disable type checking." + expr: "type(dyn([1, 'one']))" + value: { type_value: "list" } + } +} +section { + name: "int" + description: "Conversions to int." + test { + name: "uint" + expr: "int(42u)" + value: { int64_value: 42 } + } + test { + name: "uint_zero" + expr: "int(0u)" + value: { int64_value: 0 } + } + test { + name: "uint_range" + expr: "int(18446744073709551615u)" + eval_error { + errors { message: "range error" } + } + } + test { + name: "double_round_neg" + expr: "int(-123.456)" + value: { int64_value: -123 } + } + test { + name: "double_nearest" + expr: "int(1.9)" + value: { int64_value: 2 } + } + test { + name: "double_nearest_neg" + expr: "int(-7.9)" + value: { int64_value: -8 } + } + test { + name: "double_half_away_pos" + expr: "int(11.5)" + value: { int64_value: 12 } + } + test { + name: "double_half_away_neg" + expr: "int(-3.5)" + value: { int64_value: -4 } + } + test { + name: "double_range" + expr: "int(1e99)" + eval_error { + errors: { message: "range" } + } + } + test { + name: "string" + expr: "int('987')" + value: { int64_value: 987 } + } + test { + name: "timestamp" + expr: "int(timestamp('2004-09-16T23:59:59Z'))" + value: { int64_value: 1095379199 } + } +} +section { + name: "string" + description: "Conversions to string." + test { + name: "int" + expr: "string(123)" + value: { string_value: "123" } + } + test { + name: "int_neg" + expr: "string(-456)" + value: { string_value: "-456" } + } + test { + name: "uint" + expr: "string(9876u)" + value: { string_value: "9876" } + } + test { + name: "double" + expr: "string(123.456)" + value: { string_value: "123.456" } + } + test { + name: "double_hard" + expr: "string(-4.5e-3)" + value: { string_value: "-0.0045" } + } + test { + name: "bytes" + expr: "string(b'abc')" + value: { string_value: "abc" } + } + test { + name: "bytes_unicode" + expr: "string(b'\\303\\277')" + value: { string_value: "ÿ" } + } + test { + name: "bytes_invalid" + expr: "string(b'\\000\\xff')" + eval_error { + errors { message: "invalid UTF-8" } + } + } +} +section { + name: "type" + description: "Type reflection tests." + test { + name: "bool" + expr: "type(true)" + value: { type_value: "bool" } + } + test { + name: "bool_denotation" + expr: "bool" + value: { type_value: "bool" } + } + test { + name: "dyn_no_denotation" + expr: "dyn" + disable_check: true + eval_error { + errors { message: "unknown varaible" } + } + } + test { + name: "int" + expr: "type(0)" + value: { type_value: "int" } + } + test { + name: "int_denotation" + expr: "int" + value: { type_value: "int" } + } + test { + name: "eq_same" + expr: "type(true) == type(false)" + value: { bool_value: true } + } + test { + name: "uint" + expr: "type(64u)" + value: { type_value: "uint" } + } + test { + name: "uint_denotation" + expr: "uint" + value: { type_value: "uint" } + } + test { + name: "double" + expr: "type(3.14)" + value: { type_value: "double" } + } + test { + name: "double_denotation" + expr: "double" + value: { type_value: "double" } + } + test { + name: "null_type" + expr: "type(null)" + value: { type_value: "null_type" } + } + test { + name: "null_type_denotation" + expr: "null_type" + value: { type_value: "null_type" } + } + test { + name: "string" + expr: "type('foo')" + value: { type_value: "string" } + } + test { + name: "string_denotation" + expr: "string" + value: { type_value: "string" } + } + test { + name: "bytes" + expr: "type(b'\\xff')" + value: { type_value: "bytes" } + } + test { + name: "bytes_denotation" + expr: "bytes" + value: { type_value: "bytes" } + } + test { + name: "list" + expr: "type([1, 2, 3])" + value: { type_value: "list" } + } + test { + name: "list_denotation" + expr: "list" + value: { type_value: "list" } + } + test { + name: "lists_monomorphic" + expr: "type([1, 2, 3]) == type(['one', 'two', 'three'])" + value: { bool_value: true } + } + test { + name: "map" + expr: "type({4: 16})" + value: { type_value: "map" } + } + test { + name: "map_denotation" + expr: "map" + value: { type_value: "map" } + } + test { + name: "map_monomorphic" + expr: "type({'one': 1}) == type({1: 'one'})" + value: { bool_value: true } + } + test { + name: "eq_diff" + expr: "type(7) == type(7u)" + value: { bool_value: false } + } + test { + name: "neq_same" + expr: "type(0.0) != type(-0.0)" + value: { bool_value: false } + } + test { + name: "neq_diff" + expr: "type(0.0) != type(0)" + value: { bool_value: true } + } + test { + name: "meta" + expr: "type(type(7)) == type(type(7u))" + value: { bool_value: true } + } + test { + name: "type" + expr: "type(int)" + value: { type_value: "type" } + } + test { + name: "type_denotation" + expr: "type" + value: { type_value: "type" } + } + test { + name: "type_type" + expr: "type(type)" + value: { type_value: "type" } + } +} +section { + name: "uint" + description: "Conversions to uint." + test { + name: "int" + expr: "uint(1729)" + value: { uint64_value: 1729 } + } + test { + name: "int_neg" + expr: "uint(-1)" + eval_error { + errors { message: "range" } + } + } + test { + name: "double" + expr: "uint(3.14159265)" + value: { uint64_value: 3 } + } + test { + name: "double_nearest_int" + expr: "int(1.9)" + value: { int64_value: 2 } + } + test { + name: "double_nearest" + expr: "uint(1.9)" + value: { uint64_value: 2 } + } + test { + name: "double_half_away" + expr: "uint(25.5)" + value: { uint64_value: 26 } + } + test { + name: "double_range" + expr: "uint(6.022e23)" + eval_error { + errors { message: "range" } + } + } + test { + name: "string" + expr: "uint('300')" + value: { uint64_value: 300 } + } +} diff --git a/tools/Dockerfile b/tools/Dockerfile new file mode 100644 index 0000000..9498331 --- /dev/null +++ b/tools/Dockerfile @@ -0,0 +1,14 @@ +FROM golang:1.24 + +ENV HOME=/usr/cel-python +WORKDIR $HOME/tools + +# pre-copy/cache go.mod for pre-downloading dependencies and only redownloading them in subsequent builds if they change +COPY go.mod go.sum ./ +RUN go mod download + +COPY ./*.go . +RUN go build -v -o /usr/local/bin/mkgherkin ./... + +WORKDIR $HOME/textproto +ENTRYPOINT ["mkgherkin"] diff --git a/tools/README.rst b/tools/README.rst new file mode 100644 index 0000000..a2c5505 --- /dev/null +++ b/tools/README.rst @@ -0,0 +1,191 @@ +########################################## +Tools to Create the Conformance Test Suite +########################################## + +The conformance test files originte from the https://github.com/google/cel-spec.git repository. +They are all protobuf objects, serialized into ``textproto``. + +The ``pb2g.py`` and the ``mkgerkin.go`` apps translate these into Gherkin. + +Purpose +======= + +Overall, the conformance test tools work like this: + +1. Start with the ``.textproto`` source: + + .. code-block:: protobuf + + test { + name: "self_eval_int_zero" + expr: "0" + value: { int64_value: 0 } + } + +2. An Interim Gherkin is created using Go-language syntax for objects. + +3. The final Gherkin is created using Python-language syntax. + Here's the Gherkin: + + .. code-block:: gherkin + + Scenario: self_eval_int_zero + When CEL expression "0" is evaluated + # int64_value:0 + Then value is IntType(source=0) + +The Gherkin can make a bunch of test features explicit in each Scenario. +Specifically, some parameters can be set in ``Given`` clause: + +- ``Given disable_check parameter is True`` + +- ``Given type_env ... is ...`` Sets a type for a given parameter name. + +- ``Given bindings ... is ...`` Sets a value for a given parameter name. + +Operation +=========== + +The Gherkin creation is controlled by a Makefile that does several things. + +- ``make all`` checks the cel-spec repository for ``.textproto`` files, + moves them to the local ``features`` directory and converts them to ``.features`` files. + +- ``make scan`` runs the ``textproto_to_gherkin.py`` app to check syntax on the protobuf + files. This is used as a smoke test when adjusting the text protobuf grammar. + +- ``make clean-broken`` cleans up the empty ``.feature`` files. These are a consequence of + ``.textproto`` files that cannot be parsed. + +- ``make clean`` removes the ``.feature`` and ``.textproto`` files. + +The underlying command is this. + +.. code-block:: bash + + % python ../tools/pb2g.py basic.textproto -o basic.feature + +This converts a source ``.textproto`` file to an interim file, and from there to a final ``.feature`` file. + +A good way to use this is to do a checkout from https://github.com/google/cel-spec.git into +an adjacent directory. By deafult, the Makefile looks for ``../../cel-spec``. If this doesn't work +for you, set the ``CEL_SPEC_PATH`` environment variable. + +On Gherkinization +================= + +The question arises on how best to serialize descriptions of the objects created by a CEL evaluation. +We have three distinct notations to deal with: + +- Source Protobuf text in the ``.textproto`` files. + +- Final Python object used by Behave. + +- An intermediate text representation in the Gherkin that's created by a Go application. + +The idea is to write Gherkin tests that match the protobuf source closely, but can be processed by Behave without too much overhead. +(There are over 900 conformance tests; we don't want to take all day.) + +The use of a **Go** application is far simpler than trying to parse the ``.textproto`` files in Python. +The **Go** operations for expanding each ``.Section`` and ``.Test`` object into a Gherkin Scenario is very small, and narrowly focused on creating an easier-to-parse representation. + +There are three broad choices for Gherkin representation of expected results. + +- Protobuf, unmodified. ``{ int64_value: -1 }``. + This pushes the final parsing into the conformance test step definitions. + +- Text representation of the Python target object. ``celpy.celtypes.IntType(-1)``. + This is highly Python-specific and not of general use to other implementers. + +- An intermediate representation. ``Value(value_type='int64_value', source='x')``. + This is trivially produced by **Go**. + It preserves the protobuf semantics in a easy-to-parse form. + +The Python representation leverages the following class definition + +.. code-block:: python + + class Primitive(NamedTuple): + """A name: value pair.""" + type_name: Token + value_text: Token + + @property + def value(self) -> Any: + ... + + +This describes a simple object with a type name and a value. +The ``Token`` class describes the syntax used by ``Go`` when serializing objects. +This includes the following kinds of symbols: + +.. csv-table:: + + NAMESPACE,``\\[name\\]`` + BOOL,``true|false`` + NULL,``NULL_VALUE`` + STRING,``"`` or ``'``-delimited strings + FLOAT,``[+-]?\\d*\\.\\d*[Ee]?[+-]?\\d*|inf|-inf|[+-]?\\d+[Ee][+-]?\\d+`` + INT,``[+-]?\\d+`` + NAME,``[a-zA-Z]\\w+`` + WHITESPACE,The usual RE whitespace, ``\\s+`` + PUNCTUATION,Any other punctuation character, this generally includes ``{``, ``}``, ``:``, and ``,`` as delimeters in a complex structure. + +A protobuf object like ``{ int64_value: 0 }`` has punctation, name, punctuation, int, and punctuation. +The parser can transform this into a ``Primitive`` object with the ``type_name`` and ``value`` attributes. + +This can be be modeled as a simple ``0`` in the Gherkin. +For a specific subset of available types the types map directly to Python objects. +For some objects, however, there isn't a trivial correspondence. + +Here is an example of some protobuf objects and the parsed representation. + +.. csv-table:: + + "{ int64_value: 0 }","Primitive(type_name=Token(type='NAME', value='int64_value'), value_text=Token(type='INT', value='0'))",IntType(source=0) + "{ uint64_value: 0 }","Primitive(type_name=Token(type='NAME', value='uint64_value'), value_text=Token(type='INT', value='0'))",UintType(source=0) + "{ double_value: 0 }","Primitive(type_name=Token(type='NAME', value='double_value'), value_text=Token(type='INT', value='0'))",DoubleType(source=0) + "{ null_value: NULL_VALUE }","Primitive(type_name=Token(type='NAME', value='null_value'), value_text=Token(type='NULL', value='NULL_VALUE'))",None + "{ bool_value: false }","Primitive(type_name=Token(type='NAME', value='bool_value'), value_text=Token(type='BOOL', value='false'))",BoolType(source=False) + "{ string_value: """" }","Primitive(type_name=Token(type='NAME', value='string_value'), value_text=Token(type='STRING', value='""""'))",StringType(source='') + "{ bytes_value: """" }","Primitive(type_name=Token(type='NAME', value='bytes_value'), value_text=Token(type='STRING', value='""""'))",BytesType(source=b'') + +The resulting celtypes are all subclass of ``celpy.celtypes.TypeType``. + +The protobuf mappings are more complex. + + +More Complex Protobuf Definitions +--------------------------------- + +A universal ``TestAllTypes`` protobuf ``MessageType`` is used by the Dynamic tests that create protobuf objects. +It has numerous fields, but provides a handy way to define complex objects in a simple structure. + +Building the tool chain +======================= + +Run the following commands in the ``tools`` directory to create the needed Docker image. + +.. code-block:: bash + + % docker pull golang + % docker build -t mkgherkin . + +The ``Dockerfile`` will create a Docker image to run the Go application. + +The ``pb2g.py`` application will run the Docker image to do the initial conversion. + +A local ``textproto`` directory is the working directory for the interim conversion from ``.textproto`` to an interim Gherkin form. +These interim Gherkin files are then processed by the ``pb2g.py`` app to create the final ``.feature`` files for the conformance test suite. + +For reference, here's a docker command to run the container, +converting files to their interim form. + +While not necessary, you can manually commands like the following in the ``textproto`` working directory. + +.. code-block:: bash + + % docker run --rm --name mkgherkin -v .:/usr/cel-python/textproto mkgherkin *.textproto + +The output from a command like this is captured by ``pg2g.py`` and then post-processed to create the final **CEL** types. + diff --git a/tools/pb2g.py b/tools/pb2g.py index 724f099..4c72660 100644 --- a/tools/pb2g.py +++ b/tools/pb2g.py @@ -99,20 +99,23 @@ Each of these requires revising the literal value into a Python-friendly form. """ +import abc import argparse import contextlib +from dataclasses import dataclass, field +from functools import partial +import inspect import logging import math import os +from pathlib import Path import re import subprocess import sys import traceback -from dataclasses import dataclass, field -from functools import partial -from pathlib import Path -from typing import (Any, Dict, Iterable, Iterator, List, Match, NamedTuple, - Optional, Set, Tuple, Type, Union) +from typing import ( + Any, Dict, Iterable, Iterator, List, Match, NamedTuple, + Optional, Set, Tuple, Type, Union) logger = logging.getLogger("pb2g") @@ -417,62 +420,129 @@ def parse_serialized_value(tokens: Tokens) -> ParseTree: value = lookahead return Primitive(name, value) -# Placeholders for CEL types. -# We could use -# from celpy.celtypes import * +# Mocks that -- when serialized -- look like CEL types. +# We could use `from celpy.celtypes import *`. +# This approach divorces us from cel-python. -class CelType(NamedTuple): - """Placeholder for many primitive CEL types""" - source: Any +class TypeType(NamedTuple): + """ABC of CEL Type Hierarchy.""" + source: Union[Any, None] = None + seconds: Union[float, None] = None + nanos: Union[float, None] = None -class UintType(CelType): pass -class StringType(CelType): pass -class NullType(CelType): pass -class IntType(CelType): pass -class BoolType(CelType): pass + @classmethod + def cel_name(cls) -> str: + return f"celpy.celtypes.{cls.__name__}" -class BytesType(NamedTuple): - source: bytes + def __repr__(self) -> str: + if self.source is None: + return f"{self.cel_name()}()" + else: + return f"{self.cel_name()}(source={self.source!a})" + +class NoneType(TypeType): + @classmethod + def cel_name(cls) -> str: + return f"NoneType" +class PrimitiveType(TypeType): def __repr__(self) -> str: - return f"{self.__class__.__name__}(source={self.source!a})" + return f"{self.cel_name()}(source={self.source!a})" -class DoubleType(NamedTuple): - source: float +class UintType(PrimitiveType): pass +class StringType(PrimitiveType): pass +class NullType(PrimitiveType): pass +class IntType(PrimitiveType): pass +class BoolType(PrimitiveType): pass +class BytesType(PrimitiveType): + source: bytes + +class DoubleType(PrimitiveType): def __repr__(self) -> str: if math.isinf(self.source): - return f"{self.__class__.__name__}(source='{self.source}')" + return f"{self.cel_name()}(source='{self.source}')" else: - return f"{self.__class__.__name__}(source={self.source})" + return f"{self.cel_name()}(source={self.source})" -class DurationType(NamedTuple): +class DurationType(TypeType): seconds: float nanos: float -class TimestampType(NamedTuple): - source: Any + def __repr__(self) -> str: + return f"{self.cel_name()}(seconds={self.seconds:.0f}, nanos={self.nanos:.0f})" + +class TimestampType(PrimitiveType): + pass class ListType(List[Any]): """Built from Values objects.""" - pass + @classmethod + def cel_name(cls) -> str: + return f"celpy.celtypes.{cls.__name__}" class MapType(Dict[str, Any]): """Built from Entries objects.""" + @classmethod + def cel_name(cls) -> str: + return f"celpy.celtypes.{cls.__name__}" + def __repr__(self) -> str: return f"{self.__class__.__name__}({super().__repr__()})" class MessageType(Dict[str, Any]): """Built from Fields objects.""" - pass - -class TypeType(NamedTuple): - value: Any + @classmethod + def cel_name(cls) -> str: + return f"celpy.celtypes.{cls.__name__}" class CELEvalError(Exception): def __init__(self, *args: Any) -> None: super().__init__(*args) - # def repr(self): + + +# From Protobuf definitions, these are the CEL types implement them. +TYPE_NAMES = { + "google.protobuf.Any": MessageType, + "google.protubuf.Any": MessageType, # Note spelling anomaly. + "google.protobuf.BoolValue": BoolType, + "google.protobuf.BytesValue": BytesType, + "google.protobuf.DoubleValue": DoubleType, + "google.protobuf.Duration": DurationType, + "google.protobuf.FloatValue": DoubleType, + "google.protobuf.Int32Value": IntType, + "google.protobuf.Int64Value": IntType, + "google.protobuf.ListValue": ListType, + "google.protobuf.StringValue": StringType, + "google.protobuf.Struct": MessageType, + "google.protobuf.Timestamp": TimestampType, + "google.protobuf.UInt32Value": UintType, + "google.protobuf.UInt64Value": UintType, + "google.protobuf.Value": MessageType, + "type": TypeType, + "list_type": ListType, + "map_type": MapType, + "map": MapType, + "list": ListType, + "string": StringType, + "bytes": BytesType, + "bool": BoolType, + "int": IntType, + "uint": UintType, + "double": DoubleType, + "null_type": NoneType, + "STRING": StringType, + "BOOL": BoolType, + "INT64": IntType, + "UINT64": UintType, + "INT32": IntType, + "UINT32": UintType, + "BYTES": BytesType, + "DOUBLE": DoubleType, +} + +def type_value(value: str) -> type: + return TYPE_NAMES[value] # CEL testing types. This is part of the unit test framework. @@ -539,13 +609,13 @@ def debug_call_stack() -> None: 'int64_value': lambda p: IntType(detokenize(p.value_text)), 'null_value': lambda p: None, 'string_value': lambda p: StringType(detokenize(p.value_text)), - 'type_value': lambda p: TypeType(detokenize(p.value_text)), + 'type_value': lambda p: type_value(detokenize(p.value_text)), # TypeType(detokenize(p.value_text)), 'uint64_value': lambda p: UintType(detokenize(p.value_text)), 'number_value': lambda p: DoubleType(detokenize(p.value_text)), 'value': lambda p: detokenize(p.value_text), } -def map_builder(*items: ParseTree) -> CelType: +def map_builder(*items: ParseTree) -> TypeType: """Builds MapType objects from the ``entries`` clauses.""" logger.debug(f" map_builder({items!r})") entries = {} @@ -557,7 +627,7 @@ def map_builder(*items: ParseTree) -> CelType: entries[key] = value return MapType(entries) -def list_builder(*items: ParseTree) -> CelType: +def list_builder(*items: ParseTree) -> TypeType: """Builds ListType objects from the ``values`` clauses.""" logger.debug(f" list_builder({items!r})") values = [] @@ -568,7 +638,7 @@ def list_builder(*items: ParseTree) -> CelType: values.append(value) return ListType(values) -def struct_builder(*items: ParseTree) -> CelType: +def struct_builder(*items: ParseTree) -> TypeType: """ Builds MessageType objects from the ``fields`` clauses. The ``Any`` special case is taken as a type cast and ignored. @@ -595,7 +665,7 @@ def struct_builder(*items: ParseTree) -> CelType: fields[key] = value return MessageType(fields) -def duration_builder(*items: ParseTree) -> CelType: +def duration_builder(*items: ParseTree) -> TypeType: """Building duration from ``seconds`` and ``nanos`` :: @@ -608,7 +678,7 @@ def duration_builder(*items: ParseTree) -> CelType: fields[field_source.type_name.value] = value return DurationType(**fields) -def timestamp_builder(*items: ParseTree) -> CelType: +def timestamp_builder(*items: ParseTree) -> TypeType: """Building timestamp from ``seconds`` :: @@ -622,7 +692,7 @@ def timestamp_builder(*items: ParseTree) -> CelType: fields[field_source.type_name.value] = value return TimestampType(**fields) -def primitive_builder(celtype: Type[CelType], *items: ParseTree) -> CelType: +def primitive_builder(celtype: Type[TypeType], *items: ParseTree) -> TypeType: """Building from some primitive, usually a ``value``. """ # debug_call_stack() # Useful for debugging. @@ -630,7 +700,7 @@ def primitive_builder(celtype: Type[CelType], *items: ParseTree) -> CelType: item = items[0] return celtype(detokenize(item.value_text)) -def any_builder(*items: ParseTree) -> CelType: +def any_builder(*items: ParseTree) -> TypeType: """ Clandestine object_value can be hidden inside an Any object. @@ -679,7 +749,7 @@ def any_builder(*items: ParseTree) -> CelType: } -def object_builder(*items: ParseTree) -> CelType: +def object_builder(*items: ParseTree) -> TypeType: """ Build an ``object_value {}`` instance. These are more complex because -- generally -- they're protobuf message instances with many individual fields. @@ -754,7 +824,7 @@ def object_builder(*items: ParseTree) -> CelType: raise ValueError(f"What is this? object_builder({items!r})") -def error_builder(*items: ParseTree) -> CelType: +def error_builder(*items: ParseTree) -> TypeType: """ Build an error result. """ @@ -762,7 +832,7 @@ def error_builder(*items: ParseTree) -> CelType: return CELEvalError(error) -def type_builder(*items: ParseTree) -> CelType: +def type_builder(*items: ParseTree) -> TypeType: """ Build a type name for the environment. We do not traverse the entire protobuf structure. @@ -770,10 +840,12 @@ def type_builder(*items: ParseTree) -> CelType: item = items[0] logger.debug(f" type_builder({item!r})") if isinstance(item, Primitive): - return TypeType(detokenize(item.value_text)) + # Wrong... return TypeType(detokenize(item.value_text)) + return type_value(detokenize(item.value_text)) else: # TODO: Descend into the type structure - return TypeType(item.type_name.value) + # Wrong... return TypeType(item.type_name.value) + return type_value(item.type_name.value) # Top-level values seen in the output Go serialization of an object. @@ -794,7 +866,7 @@ def type_builder(*items: ParseTree) -> CelType: } -def structure_builder(structure: ParseTree) -> CelType: +def structure_builder(structure: ParseTree) -> TypeType: """ Top-level generic builder of CEL objects from the serialized Go object. """ @@ -813,121 +885,185 @@ def structure_builder(structure: ParseTree) -> CelType: else: raise ValueError(f"What is this? structure_builder({structure!r})") +class Gherkinizer(abc.ABC): + """Abstract Base Class for Gherkinization.""" -def gherkinize(gherkinizer_path: Path, source_path: Optional[Path], target_path: Optional[Path]) -> None: - """ - Convert from textproto to Gherkin that contains Go value serializations. + @abc.abstractmethod + def convert(self, source_path: Optional[Path], target_path: Optional[Path]) -> None: + ... - Requires GO on the Path. +class LocalGherkinzer(Gherkinizer): + GHERKINIZER = Path.cwd() / "tools" / "mkgherkin.go" - :: + def convert(self, source_path: Optional[Path], target_path: Optional[Path]) -> None: + """ + Convert from textproto to interim Gherkin that contains Go value serializations. - go mod init mkgherkin - go mod tidy + Requires GO on the ``PATH``. - :: + Build the app: + .. code-block:: bash - export PATH="/usr/local/go/bin:/usr/local/bin:$PATH" - export GOPATH="~/go" - """ - logger.info(f"With {gherkinizer_path}") - if source_path: - logger.info(f"gherkinize {source_path.name} -> {target_path.name}") - - env = { - "PATH": f"/usr/local/go/bin:/usr/local/bin:{os.environ['PATH']}", - "GOPATH": str(Path.home()/"go"), - "HOME": str(Path.home()), - } - command = ["go", "run", gherkinizer_path.stem] - if source_path: - command.append(str(source_path.absolute())) - - output_context = target_path.open('w') if target_path else contextlib.nullcontext(None) - try: - with output_context as interim: - subprocess.run( - command, - env=env, - cwd=gherkinizer_path.parent, - stdout=interim, - check=True) - except subprocess.CalledProcessError as ex: - print( - f"{' '.join(command)} failed; " - f"perhaps `go mod init {gherkinizer_path.stem}; go get` is required." - ) - raise + % go mod init mkgherkin + % go mod tidy + + Run the app: + .. code-block:: bash + + % export PATH="/usr/local/go/bin:/usr/local/bin:$PATH" + % export GOPATH="~/go" + % mkgherkin.go name.textproto >name.interim + """ + logger.info(f"With local {self.GHERKINIZER}") + if source_path and target_path: + logger.info(f"gherkinize {source_path.name} -> {target_path.name}") + elif source_path: + logger.info(f"gherkinize {source_path.name} -> STDOUT") + + env = { + "PATH": f"/usr/local/go/bin:/usr/local/bin:{os.environ['PATH']}", + "GOPATH": str(Path.home()/"go"), + "HOME": str(Path.home()), + } + command = ["go", "run", self.GHERKINIZER.stem] + + if source_path: + command.append(str(source_path.absolute())) + + output_context = target_path.open('w') if target_path else contextlib.nullcontext(None) + try: + with output_context as interim: + subprocess.run( + command, + env=env, + stdout=interim, + check=True) + except subprocess.CalledProcessError as ex: + print( + f"{' '.join(command)} failed; " + f"perhaps `go mod init {self.GHERKINIZER.stem}; go get` is required." + ) + raise + +class DockerGherkinzer(Gherkinizer): + def convert(self, source_path: Optional[Path], target_path: Optional[Path]) -> None: + """ + Convert from textproto to interim Gherkin that contains Go value serializations. + + Requires ``docker`` on the ``PATH``. + + See ``tools/Dockerfile``. + + Build the docker image: + .. code-block:: bash + + % docker pull golang + % docker build -t mkgherkin . + + Run the docker image: + .. code-block:: bash + + % docker run --rm --name mkgherkin -v .:/usr/cel-python/textproto mkgherkin name.textproto >name.interim + """ + logger.info("With 'mkgherkin' Docker image") + if source_path and target_path: + logger.info(f"gherkinize {source_path.name} -> {target_path.name}") + elif source_path: + logger.info(f"gherkinize {source_path.name} -> STDOUT") + + command = ["docker", "run", "--rm", "--name", "mkgherkin", "-v", ".:/usr/cel-python/textproto", "mkgherkin"] + if source_path: + command.append(str(source_path.name)) + + try: + output_context = ( + target_path.open('w') if target_path else contextlib.nullcontext(None) + ) + with output_context as interim: + subprocess.run( + command, + stdout=interim, + check=True) + except subprocess.CalledProcessError as ex: + print( + f"{' '.join(command)} failed; " + f"`docker` and mkgherkin docker image are required." + ) + raise + +def expand_cel(feature_text: str) -> None: + """Revises CEL from Go notation to Python notation.""" + given_bindings_pat = re.compile(r'\s*Given bindings parameter "(.*?)" is\s+(.*)') + given_type_env_pat = re.compile(r'\s*Given type_env parameter "(.*?)" is\s+&\{(.*)\}') + when_expr_pat = re.compile(r'\s*When CEL expression "(.*)" is evaluated') + then_value_pat = re.compile(r"\s*Then value is\s+(.*)") + then_error_pat = re.compile(r"\s*Then eval_error is\s+(.*)") + + for line in feature_text.splitlines(): + given_binding_line = given_bindings_pat.match(line) + given_type_env_line = given_type_env_pat.match(line) + when_expr_line = when_expr_pat.match(line) + then_value_line = then_value_pat.match(line) + then_error_line = then_error_pat.match(line) + if given_binding_line: + # Replace the value with a proper CEL object + variable, value = given_binding_line.groups() + replacement = structure_builder(parse_serialized_value(Tokens(value))) + print(f' # {value}') + print(f' Given bindings parameter "{variable}" is {replacement}') + elif given_type_env_line: + # Replace the value with a CEL-friendly variant on the type name + variable, type_spec = given_type_env_line.groups() + replacement = structure_builder(parse_serialized_value(Tokens(type_spec))) + print(f' # {type_spec}') + print(f' Given type_env parameter "{variable}" is {replacement.cel_name()}') + elif then_value_line: + # Replace the value with a proper CEL object + value = then_value_line.group(1) + replacement = structure_builder(parse_serialized_value(Tokens(value))) + print(f' # {value}') + if type(replacement) == type: + print(f' Then value is {replacement.cel_name()}') + else: + print(f' Then value is {replacement!r}') + elif then_error_line: + # Replace the error with a more useful CEL-like. + value = then_error_line.group(1) + replacement_exception = structure_builder(parse_serialized_value(Tokens(value))) + print(f' # {value}') + print(f' Then eval_error is {replacement_exception.args[0]!r}') + elif when_expr_line: + # Clean up escaped quotes within the CEL expr. + value = when_expr_line.group(1) + replacement = ''.join( + expand_str_escape(m.group()) for m in STR_ESCAPES.finditer(value)) + if '"' in replacement: + print(f" When CEL expression '{replacement}' is evaluated") + else: + print(f' When CEL expression "{replacement}" is evaluated') + else: + # it's already perfect + print(line) def celify(source_path: Path, target_path: Optional[Path]) -> None: """ - Rewrite Gherkin that contains Go value serializations into Gherkin with CEL serializations. + Rewrite interim Gherkin that contains Go value serializations into final Gherkin with CEL serializations. - This reads the intermediate Gherkin, looking for specific clauses with serialized values: + This reads the interim Gherkin, looking for specific clauses with serialized values: - Given bindings parameter ... is ... - Given type_env parameter ... is ... - When CEL expression "..." is evaluated - Then value is ... - Both of these contain Go values which are parsed and rebuilt as CEL objects. - The resulting Gherkin is written to stdout, which may be redirected to a file. + These contain Go values which are parsed and rebuilt as CEL objects. + The resulting Gherkin is written to a given Path, or stdout if not Path is given. """ if target_path: logger.info(f"celify {source_path.name} -> {target_path.name}") else: logger.info(f"celify {source_path.name}") - def expand_cel(feature_text: str): - bindings_pat = re.compile(r'\s*Given bindings parameter "(.*?)" is\s+(.*)') - type_env_pat = re.compile(r'\s*Given type_env parameter "(.*?)" is\s+&\{(.*)\}') - when_expr_pat = re.compile(r'\s*When CEL expression "(.*)" is evaluated') - then_value_pat = re.compile(r"\s*Then value is\s+(.*)") - then_error_pat = re.compile(r"\s*Then eval_error is\s+(.*)") - - for line in feature_text.splitlines(): - binding_line = bindings_pat.match(line) - type_env_line = type_env_pat.match(line) - when_expr_line = when_expr_pat.match(line) - then_value_line = then_value_pat.match(line) - then_error_line = then_error_pat.match(line) - if binding_line: - # Replace the value with a proper CEL object - variable, value = binding_line.groups() - replacement = structure_builder(parse_serialized_value(Tokens(value))) - print(f' # {value}') - print(f' Given bindings parameter "{variable}" is {replacement}') - elif type_env_line: - # Replace the value with a CEL-friendly variant on the type name - variable, type_spec = type_env_line.groups() - replacement = structure_builder(parse_serialized_value(Tokens(type_spec))) - print(f' # {type_spec}') - print(f' Given type_env parameter "{variable}" is {replacement}') - elif then_value_line: - # Replace the value with a proper CEL object - value = then_value_line.group(1) - replacement = structure_builder(parse_serialized_value(Tokens(value))) - print(f' # {value}') - print(f' Then value is {replacement!r}') - elif then_error_line: - # Replace the error with a more useful CEL-like. - value = then_error_line.group(1) - replacement_exception = structure_builder(parse_serialized_value(Tokens(value))) - print(f' # {value}') - print(f' Then eval_error is {replacement_exception.args[0]!r}') - elif when_expr_line: - # Clean up escaped quotes within the CEL expr. - value = when_expr_line.group(1) - replacement = ''.join( - expand_str_escape(m.group()) for m in STR_ESCAPES.finditer(value)) - if '"' in replacement: - print(f" When CEL expression '{replacement}' is evaluated") - else: - print(f' When CEL expression "{replacement}" is evaluated') - else: - # it's already perfect - print(line) - feature_text = source_path.read_text() if target_path: with target_path.open('w') as target_file: @@ -941,9 +1077,9 @@ def expand_cel(feature_text: str): def get_options(argv: List[str] = sys.argv[1:]) -> argparse.Namespace: parser = argparse.ArgumentParser() parser.add_argument( - '-g', '--gherkinizer', action='store', type=Path, - help="Location of the mkgherkin.go module", - default=Path(__file__).parent / "mkgherkin.go") + '-g', '--gherkinizer', + choices=["docker", "local"], + default="docker") parser.add_argument( '-v', '--verbose', dest="log_level", @@ -969,14 +1105,16 @@ def get_options(argv: List[str] = sys.argv[1:]) -> argparse.Namespace: options = get_options() logging.getLogger().setLevel(options.log_level) - mkgherkin = options.gherkinizer + if options.gherkinizer.lower() == "docker": + gherkinzer = DockerGherkinzer() + else: + gherkinzer = LocalGherkinzer() if not options.source: - gherkinize(mkgherkin, None, None) + gherkinzer.convert(None, None) sys.exit() source = options.source - interim = (Path.cwd()/f"{source.stem}_go").with_suffix(".gherkin") - gherkinize(mkgherkin, source, interim) + interim = source.with_suffix(".interim") + gherkinzer.convert(source, interim) celify(interim, options.output) - interim.unlink() diff --git a/tools/test_pb2g.py b/tools/test_pb2g.py index 241861d..84943b8 100644 --- a/tools/test_pb2g.py +++ b/tools/test_pb2g.py @@ -16,7 +16,9 @@ """ Test translation of the pb2g tool to convert textproto to Gherkin.. """ -from pytest import * +from textwrap import dedent + +import pytest import pb2g from pb2g import * @@ -50,6 +52,88 @@ def test_building_case(): assert s == expected, f"{s!r} != {expected!r}" +def test_transform_given_type(capsys): + proto = dedent("""\ + section { + name: "variables" + description: "Variable lookups." + test { + name: "self_eval_bound_lookup" + expr: "x" + type_env: { + name: "x", + ident: { type: { primitive: INT64 } } + } + bindings: { + key: "x" + value: { value: { int64_value: 123 } } + } + value: { int64_value: 123 } + } + } + """ + ) + interim = dedent("""\ + Scenario: self_eval_bound_lookup + + Given type_env parameter "x" is &{type:{primitive:INT64}} + + Given bindings parameter "x" is int64_value:123 + + When CEL expression "x" is evaluated + Then value is int64_value:123 + """) + + expected = dedent("""\ + Scenario: self_eval_bound_lookup + + # type:{primitive:INT64} + Given type_env parameter "x" is celpy.celtypes.IntType + + # int64_value:123 + Given bindings parameter "x" is celpy.celtypes.IntType(source=123) + + When CEL expression "x" is evaluated + # int64_value:123 + Then value is celpy.celtypes.IntType(source=123) + """) + + pb2g.expand_cel(interim) + feature, errors = capsys.readouterr() + assert errors == "" + assert feature == expected + + +def test_transform_then_type(capsys): + proto = dedent("""\ + test { + name: "type_type" + expr: "type(type)" + value: { type_value: "type" } + } + """ + ) + interim = dedent("""\ + Scenario: type_type + + When CEL expression "type(type)" is evaluated + Then value is type_value:"type" + """) + + expected = dedent("""\ + Scenario: type_type + + When CEL expression "type(type)" is evaluated + # type_value:"type" + Then value is celpy.celtypes.TypeType + """) + + pb2g.expand_cel(interim) + feature, errors = capsys.readouterr() + assert errors == "" + assert feature == expected + + def test_given_bindings(): assert structure_builder(parse_serialized_value(Tokens('value:{int64_value:123}'))) == IntType(source=123) assert structure_builder(parse_serialized_value(Tokens('value:{bool_value:true}'))) == BoolType(source=True) @@ -212,13 +296,21 @@ def test_given_bindings(): seconds=123, nanos=123456789) -@fixture +@pytest.fixture def exception_equal(monkeypatch): def args_eq(exc1, exc2): return type(exc1) == type(exc2) and exc1.args == exc2.args monkeypatch.setattr(pb2g.CELEvalError, '__eq__', args_eq) +def test_parsing_primitives(): + assert (parse_serialized_value(Tokens('int64_value:0'))) == Primitive(type_name=Token(type='NAME', value='int64_value'), value_text=Token(type='INT', value='0')) + assert (parse_serialized_value(Tokens('uint64_value:0'))) == Primitive(type_name=Token(type='NAME', value='uint64_value'), value_text=Token(type='INT', value='0')) + assert (parse_serialized_value(Tokens('double_value:0'))) == Primitive(type_name=Token(type='NAME', value='double_value'), value_text=Token(type='INT', value='0')) + assert (parse_serialized_value(Tokens('string_value:""'))) == Primitive(type_name=Token(type='NAME', value='string_value'), value_text=Token(type='STRING', value='""')) + assert (parse_serialized_value(Tokens('bytes_value:""'))) == Primitive(type_name=Token(type='NAME', value='bytes_value'), value_text=Token(type='STRING', value='""')) + assert (parse_serialized_value(Tokens('bool_value:false'))) == Primitive(type_name=Token(type='NAME', value='bool_value'), value_text=Token(type='BOOL', value='false')) + assert (parse_serialized_value(Tokens('null_value:NULL_VALUE'))) == Primitive(type_name=Token(type='NAME', value='null_value'), value_text=Token(type='NULL', value='NULL_VALUE')) def test_then_values(exception_equal): assert structure_builder(parse_serialized_value(Tokens('int64_value:0'))) == IntType(source=0) @@ -293,7 +385,7 @@ def test_then_values(exception_equal): source=-843200000.0) assert structure_builder(parse_serialized_value(Tokens('double_value:-5.43e-21'))) == DoubleType( source=-5.43e-21) - assert structure_builder(parse_serialized_value(Tokens('type_value:"list"'))) == TypeType(value='list') + assert structure_builder(parse_serialized_value(Tokens('type_value:"list"'))) == type_value(value='list') assert structure_builder(parse_serialized_value(Tokens('errors:{message:"range error"}'))) == CELEvalError( 'range error') assert structure_builder(parse_serialized_value(Tokens('int64_value:-123'))) == IntType(source=-123) @@ -316,19 +408,19 @@ def test_then_values(exception_equal): assert structure_builder(parse_serialized_value(Tokens('string_value:"ÿ"'))) == StringType(source='ÿ') assert structure_builder(parse_serialized_value(Tokens('errors:{message:"invalid UTF-8"}'))) == CELEvalError( 'invalid UTF-8') - assert structure_builder(parse_serialized_value(Tokens('type_value:"bool"'))) == TypeType(value='bool') + assert structure_builder(parse_serialized_value(Tokens('type_value:"bool"'))) == type_value(value='bool') assert structure_builder( parse_serialized_value(Tokens('errors:{message:"unknown varaible"}'))) == CELEvalError( 'unknown varaible') - assert structure_builder(parse_serialized_value(Tokens('type_value:"int"'))) == TypeType(value='int') - assert structure_builder(parse_serialized_value(Tokens('type_value:"uint"'))) == TypeType(value='uint') - assert structure_builder(parse_serialized_value(Tokens('type_value:"double"'))) == TypeType(value='double') - assert structure_builder(parse_serialized_value(Tokens('type_value:"null_type"'))) == TypeType( + assert structure_builder(parse_serialized_value(Tokens('type_value:"int"'))) == type_value(value='int') + assert structure_builder(parse_serialized_value(Tokens('type_value:"uint"'))) == type_value(value='uint') + assert structure_builder(parse_serialized_value(Tokens('type_value:"double"'))) == type_value(value='double') + assert structure_builder(parse_serialized_value(Tokens('type_value:"null_type"'))) == type_value( value='null_type') - assert structure_builder(parse_serialized_value(Tokens('type_value:"string"'))) == TypeType(value='string') - assert structure_builder(parse_serialized_value(Tokens('type_value:"bytes"'))) == TypeType(value='bytes') - assert structure_builder(parse_serialized_value(Tokens('type_value:"map"'))) == TypeType(value='map') - assert structure_builder(parse_serialized_value(Tokens('type_value:"type"'))) == TypeType(value='type') + assert structure_builder(parse_serialized_value(Tokens('type_value:"string"'))) == type_value(value='string') + assert structure_builder(parse_serialized_value(Tokens('type_value:"bytes"'))) == type_value(value='bytes') + assert structure_builder(parse_serialized_value(Tokens('type_value:"map"'))) == type_value(value='map') + assert structure_builder(parse_serialized_value(Tokens('type_value:"type"'))) == type_value(value='type') assert structure_builder(parse_serialized_value(Tokens('uint64_value:1729'))) == UintType(source=1729) assert structure_builder(parse_serialized_value(Tokens('uint64_value:3'))) == UintType(source=3) assert structure_builder(parse_serialized_value(Tokens('uint64_value:2'))) == UintType(source=2) @@ -2169,12 +2261,12 @@ def test_then_values_2(exception_equal): parse_serialized_value(Tokens('string_value:"2009-02-13T23:31:30Z"'))) == StringType( source='2009-02-13T23:31:30Z') assert structure_builder( - parse_serialized_value(Tokens('type_value:"google.protobuf.Timestamp"'))) == TypeType( + parse_serialized_value(Tokens('type_value:"google.protobuf.Timestamp"'))) == type_value( value='google.protobuf.Timestamp') assert structure_builder(parse_serialized_value(Tokens('string_value:"1000000s"'))) == StringType( source='1000000s') assert structure_builder( - parse_serialized_value(Tokens('type_value:"google.protobuf.Duration"'))) == TypeType( + parse_serialized_value(Tokens('type_value:"google.protobuf.Duration"'))) == type_value( value='google.protobuf.Duration') assert structure_builder(parse_serialized_value(Tokens('int64_value:13'))) == IntType(source=13) assert structure_builder(parse_serialized_value(Tokens('int64_value:43'))) == IntType(source=43) @@ -2189,16 +2281,30 @@ def test_then_values_2(exception_equal): assert structure_builder(parse_serialized_value(Tokens('int64_value:3730'))) == IntType(source=3730) def test_type_env_values(): - assert structure_builder(parse_serialized_value(Tokens('type:{message_type:"google.protobuf.Int32Value"}'))) == TypeType(value='google.protobuf.Int32Value') - assert structure_builder(parse_serialized_value(Tokens('type:{message_type:"google.protobuf.Int64Value"}'))) == TypeType(value='google.protobuf.Int64Value') - assert structure_builder(parse_serialized_value(Tokens('type:{message_type:"google.protobuf.UInt32Value"}'))) == TypeType(value='google.protobuf.UInt32Value') - assert structure_builder(parse_serialized_value(Tokens('type:{message_type:"google.protobuf.UInt64Value"}'))) == TypeType(value='google.protobuf.UInt64Value') - assert structure_builder(parse_serialized_value(Tokens('type:{message_type:"google.protobuf.FloatValue"}'))) == TypeType(value='google.protobuf.FloatValue') - assert structure_builder(parse_serialized_value(Tokens('type:{message_type:"google.protobuf.DoubleValue"}'))) == TypeType(value='google.protobuf.DoubleValue') - assert structure_builder(parse_serialized_value(Tokens('type:{message_type:"google.protobuf.BoolValue"}'))) == TypeType(value='google.protobuf.BoolValue') - assert structure_builder(parse_serialized_value(Tokens('type:{message_type:"google.protobuf.StringValue"}'))) == TypeType(value='google.protobuf.StringValue') - assert structure_builder(parse_serialized_value(Tokens('type:{message_type:"google.protobuf.BytesValue"}'))) == TypeType(value='google.protobuf.BytesValue') - assert structure_builder(parse_serialized_value(Tokens('type:{message_type:"google.protobuf.ListValue"}'))) == TypeType(value='google.protobuf.ListValue') - assert structure_builder(parse_serialized_value(Tokens('type:{message_type:"google.protobuf.Struct"}'))) == TypeType(value='google.protobuf.Struct') - assert structure_builder(parse_serialized_value(Tokens('type:{message_type:"google.protobuf.Value"}'))) == TypeType(value='google.protobuf.Value') - assert structure_builder(parse_serialized_value(Tokens('type:{message_type:"google.protubuf.Any"}'))) == TypeType(value='google.protubuf.Any') + assert structure_builder(parse_serialized_value(Tokens('type:{message_type:"google.protobuf.Int32Value"}'))) == type_value(value='google.protobuf.Int32Value') + assert structure_builder(parse_serialized_value(Tokens('type:{message_type:"google.protobuf.Int64Value"}'))) == type_value(value='google.protobuf.Int64Value') + assert structure_builder(parse_serialized_value(Tokens('type:{message_type:"google.protobuf.UInt32Value"}'))) == type_value(value='google.protobuf.UInt32Value') + assert structure_builder(parse_serialized_value(Tokens('type:{message_type:"google.protobuf.UInt64Value"}'))) == type_value(value='google.protobuf.UInt64Value') + assert structure_builder(parse_serialized_value(Tokens('type:{message_type:"google.protobuf.FloatValue"}'))) == type_value(value='google.protobuf.FloatValue') + assert structure_builder(parse_serialized_value(Tokens('type:{message_type:"google.protobuf.DoubleValue"}'))) == type_value(value='google.protobuf.DoubleValue') + assert structure_builder(parse_serialized_value(Tokens('type:{message_type:"google.protobuf.BoolValue"}'))) == type_value(value='google.protobuf.BoolValue') + assert structure_builder(parse_serialized_value(Tokens('type:{message_type:"google.protobuf.StringValue"}'))) == type_value(value='google.protobuf.StringValue') + assert structure_builder(parse_serialized_value(Tokens('type:{message_type:"google.protobuf.BytesValue"}'))) == type_value(value='google.protobuf.BytesValue') + assert structure_builder(parse_serialized_value(Tokens('type:{message_type:"google.protobuf.ListValue"}'))) == type_value(value='google.protobuf.ListValue') + assert structure_builder(parse_serialized_value(Tokens('type:{message_type:"google.protobuf.Struct"}'))) == type_value(value='google.protobuf.Struct') + assert structure_builder(parse_serialized_value(Tokens('type:{message_type:"google.protobuf.Value"}'))) == type_value(value='google.protobuf.Value') + assert structure_builder(parse_serialized_value(Tokens('type:{message_type:"google.protubuf.Any"}'))) == type_value(value='google.protubuf.Any') + +def test_type_repr(): + assert type_value("type").cel_name() == "celpy.celtypes.TypeType" + assert type_value("bool").cel_name() == "celpy.celtypes.BoolType" + assert type_value("bytes").cel_name() == "celpy.celtypes.BytesType" + assert type_value("double").cel_name() == "celpy.celtypes.DoubleType" + assert type_value("int").cel_name() == "celpy.celtypes.IntType" + assert type_value("list").cel_name() == "celpy.celtypes.ListType" + assert type_value("list_type").cel_name() == "celpy.celtypes.ListType" + assert type_value("map").cel_name() == "celpy.celtypes.MapType" + assert type_value("map_type").cel_name() == "celpy.celtypes.MapType" + assert type_value("null_type").cel_name() == "NoneType" + assert type_value("string").cel_name() == "celpy.celtypes.StringType" + assert type_value("uint").cel_name() == "celpy.celtypes.UintType" diff --git a/tox.ini b/tox.ini index 8f300eb..83c4394 100644 --- a/tox.ini +++ b/tox.ini @@ -25,10 +25,11 @@ deps = setenv = PYTHONPATH = {toxinidir}/src commands = - pytest -v + pytest -vv pytest -v --doctest-modules src pytest -v --doctest-modules README.rst - behave --tags=~@wip -D env='{envname}' --stop + behave --tags=~@wip --tags=~@future -D env='{envname}' --stop + behave --tags=~@wip --tags=~@future -D env='{envname}' -D runner='compiled' --stop python -m doctest {toxinidir}/docs/source/api.rst {toxinidir}/docs/source/cli.rst {toxinidir}/docs/source/index.rst {toxinidir}/docs/source/integration.rst pytest -v -o python_classes='PYTest*' tools @@ -55,7 +56,7 @@ deps = tomli commands = coverage erase - pytest -vv --cov=src --cov-report=term-missing + pytest -vv -x --cov=src --cov-report=term-missing coverage html coverage xml mypy --lineprecision-report type_check --strict --disable-error-code type-arg --show-error-codes src diff --git a/type_check/lineprecision.txt b/type_check/lineprecision.txt index 94950e0..1aded60 100644 --- a/type_check/lineprecision.txt +++ b/type_check/lineprecision.txt @@ -1,11 +1,11 @@ Name Lines Precise Imprecise Any Empty Unanalyzed ------------------------------------------------------------------- -celpy 306 65 11 4 226 0 -celpy.__main__ 538 209 13 43 273 0 -celpy.adapter 141 34 3 9 89 6 -celpy.c7nlib 1663 349 16 149 1149 0 -celpy.celparser 390 137 67 23 163 0 -celpy.celtypes 1504 417 14 224 812 37 -celpy.evaluation 2603 661 234 198 1496 14 +celpy 306 79 12 4 211 0 +celpy.__main__ 549 243 14 60 232 0 +celpy.adapter 142 34 3 9 90 6 +celpy.c7nlib 1663 344 16 152 1151 0 +celpy.celparser 411 138 67 23 183 0 +celpy.celtypes 1480 385 15 234 809 37 +celpy.evaluation 3873 1136 249 244 2228 16 xlate 0 0 0 0 0 0 -xlate.c7n_to_cel 1755 395 103 146 1105 6 +xlate.c7n_to_cel 1755 397 103 144 1105 6 diff --git a/uv.lock b/uv.lock index 9e6aed4..5464d5c 100644 --- a/uv.lock +++ b/uv.lock @@ -73,9 +73,6 @@ dependencies = [ { name = "lark" }, { name = "pendulum" }, { name = "pyyaml" }, - { name = "sphinx", version = "7.4.7", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.10'" }, - { name = "sphinx", version = "8.1.3", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version == '3.10.*'" }, - { name = "sphinx", version = "8.2.3", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11'" }, { name = "tomli", marker = "python_full_version < '3.11'" }, ] @@ -92,6 +89,10 @@ dev = [ { name = "mypy" }, { name = "pytest" }, { name = "ruff" }, + { name = "sphinx", version = "7.4.7", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.10'" }, + { name = "sphinx", version = "8.1.3", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version == '3.10.*'" }, + { name = "sphinx", version = "8.2.3", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11'" }, + { name = "sphinxcontrib-plantuml" }, { name = "tox" }, { name = "tox-uv" }, { name = "types-pyyaml" }, @@ -104,7 +105,6 @@ requires-dist = [ { name = "lark", specifier = ">=1.2.2" }, { name = "pendulum", specifier = ">=3.1.0" }, { name = "pyyaml", specifier = ">=6.0.2" }, - { name = "sphinx", specifier = ">=7.4.7" }, { name = "tomli", marker = "python_full_version < '3.11'", specifier = ">=1.1.0" }, ] provides-extras = ["re2"] @@ -117,6 +117,8 @@ dev = [ { name = "mypy", specifier = ">=1.15.0" }, { name = "pytest", specifier = ">=8.3.5" }, { name = "ruff", specifier = ">=0.11.10" }, + { name = "sphinx", specifier = ">=7.4.7" }, + { name = "sphinxcontrib-plantuml", specifier = ">=0.30" }, { name = "tox", specifier = ">=4.24" }, { name = "tox-uv", specifier = ">=1.25.0" }, { name = "types-pyyaml", specifier = ">=6.0.12.20250516" }, @@ -1007,6 +1009,17 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/c2/42/4c8646762ee83602e3fb3fbe774c2fac12f317deb0b5dbeeedd2d3ba4b77/sphinxcontrib_jsmath-1.0.1-py2.py3-none-any.whl", hash = "sha256:2ec2eaebfb78f3f2078e73666b1415417a116cc848b72e5172e596c871103178", size = 5071, upload-time = "2019-01-21T16:10:14.333Z" }, ] +[[package]] +name = "sphinxcontrib-plantuml" +version = "0.30" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "sphinx", version = "7.4.7", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.10'" }, + { name = "sphinx", version = "8.1.3", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version == '3.10.*'" }, + { name = "sphinx", version = "8.2.3", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/4d/e0/91ee50f1a020e2ed48d370a054f94b012ba0d757214a420ac43c9327f818/sphinxcontrib-plantuml-0.30.tar.gz", hash = "sha256:2a1266ca43bddf44640ae44107003df4490de2b3c3154a0d627cfb63e9a169bf", size = 15084, upload-time = "2024-05-22T13:34:43.339Z" } + [[package]] name = "sphinxcontrib-qthelp" version = "2.0.0"