diff --git a/.gitignore b/.gitignore
index 34a9a33..16e68fa 100644
--- a/.gitignore
+++ b/.gitignore
@@ -134,3 +134,6 @@ type_check/lineprecision.txt
demo/issue_41.py
demo/issue_80.py
setup.sh
+/docs/build/doctrees
+docs/build/html/.buildinfo
+docs/build/html/.buildinfo.bak
diff --git a/.travis.yml b/.travis.yml
deleted file mode 100644
index 9e26bf7..0000000
--- a/.travis.yml
+++ /dev/null
@@ -1,21 +0,0 @@
-language: python
-cache: pip
-
-matrix:
- include:
- - python: 3.6
- env: TOXENV=py36
- - python: 3.7
- sudo: true
- env: TOXENV=py37
- - python: 3.8
- sudo: true
- env: TOXENV=py38
-
-install:
- - pip install tox
-
-script:
- - tox
-
-dist: xenial
diff --git a/Makefile b/Makefile
index a6842b8..acad1b9 100644
--- a/Makefile
+++ b/Makefile
@@ -21,35 +21,30 @@ build:
uv build
install-tools:
- cd tools && export PATH="/usr/local/go/bin:/usr/local/bin:$PATH" && go mod init mkgherkin && go mod tidy
+ cd tools && docker pull golang && docker build -t mkgherkin .
test:
cd features && $(MAKE) all
- tox -e py312
+ tox run -e py312
test-all:
cd features && $(MAKE) all
- tox
+ tox run
test-wip:
cd features && $(MAKE) all
- tox -e wip
+ tox run -e wip
test-tools:
- tox -e tools
+ tox run -e tools
cd features && $(MAKE) scan
-unit-test:
- PYTHONPATH=src python -m pytest -vv --cov=src --cov-report=term-missing ${test}
- PYTHONPATH=src python -m doctest tools/*.py
- PYTHONPATH=src python -m doctest features/steps/*.py
-
docs: $(wildcard docs/source/*.rst)
PYTHONPATH=src python -m doctest docs/source/*.rst
export PYTHONPATH=$(PWD)/src:$(PWD)/tools && cd docs && $(MAKE) html
lint:
- tox -e lint
+ tox run -e lint
coverage:
coverage report -m
diff --git a/README.rst b/README.rst
index 666a97c..ae44fd5 100644
--- a/README.rst
+++ b/README.rst
@@ -128,11 +128,11 @@ can then be applied to argument values.
>>> ast = env.compile(cel_source)
>>> prgm = env.program(ast)
- >>> activation = {
+ >>> context = {
... "account": celpy.json_to_cel({"balance": 500, "overdraftProtection": False}),
... "transaction": celpy.json_to_cel({"withdrawal": 600})
... }
- >>> result = prgm.evaluate(activation)
+ >>> result = prgm.evaluate(context)
>>> result
BoolType(False)
@@ -153,10 +153,15 @@ https://github.com/google/cel-cpp/blob/master/parser/Cel.g4
https://github.com/google/cel-go/blob/master/parser/gen/CEL.g4
+The documentation includes PlantUML diagrams.
+The Sphinx ``conf.py`` provides the location for the PlantUML local JAR file if one is used.
+Currently, it expects ``docs/plantuml-asl-1.2025.3.jar``.
+The JAR is not provided in this repository, get one from https://plantuml.com.
+If you install a different version, update the ``conf.py`` to refer to the JAR file you've downloaded.
+
Notes
=====
-
CEL provides a number of runtime errors that are mapped to Python exceptions.
- ``no_matching_overload``: this function has no overload for the types of the arguments.
@@ -171,6 +176,79 @@ However, see https://github.com/google/cel-spec/blob/master/doc/langdef.md#gradu
Rather than try to pre-check types, we'll rely on Python's implementation.
+Example 2
+=========
+
+Here's an example with some details::
+
+ >>> import celpy
+
+ # A list of type names and class bindings used to create an environment.
+ >>> types = []
+ >>> env = celpy.Environment(types)
+
+ # Parse the code to create the CEL AST.
+ >>> ast = env.compile("355. / 113.")
+
+ # Use the AST and any overriding functions to create an executable program.
+ >>> functions = {}
+ >>> prgm = env.program(ast, functions)
+
+ # Variable bindings.
+ >>> activation = {}
+
+ # Final evaluation.
+ >>> try:
+ ... result = prgm.evaluate(activation)
+ ... error = None
+ ... except CELEvalError as ex:
+ ... result = None
+ ... error = ex.args[0]
+
+ >>> result # doctest: +ELLIPSIS
+ DoubleType(3.14159...)
+
+Example 3
+=========
+
+See https://github.com/google/cel-go/blob/master/examples/simple_test.go
+
+The model Go we're sticking close to::
+
+ d := cel.Declarations(decls.NewVar("name", decls.String))
+ env, err := cel.NewEnv(d)
+ if err != nil {
+ log.Fatalf("environment creation error: %v\\n", err)
+ }
+ ast, iss := env.Compile(`"Hello world! I'm " + name + "."`)
+ // Check iss for compilation errors.
+ if iss.Err() != nil {
+ log.Fatalln(iss.Err())
+ }
+ prg, err := env.Program(ast)
+ if err != nil {
+ log.Fatalln(err)
+ }
+ out, _, err := prg.Eval(map[string]interface{}{
+ "name": "CEL",
+ })
+ if err != nil {
+ log.Fatalln(err)
+ }
+ fmt.Println(out)
+ // Output:Hello world! I'm CEL.
+
+Here's the Pythonic approach, using concept patterned after the Go implementation::
+
+ >>> from celpy import *
+ >>> decls = {"name": celtypes.StringType}
+ >>> env = Environment(annotations=decls)
+ >>> ast = env.compile('"Hello world! I\'m " + name + "."')
+ >>> out = env.program(ast).evaluate({"name": "CEL"})
+ >>> print(out)
+ Hello world! I'm CEL.
+
+
Contributing
============
diff --git a/benches/complex_expression.py b/benches/complex_expression.py
index 7f1fc49..29aa269 100644
--- a/benches/complex_expression.py
+++ b/benches/complex_expression.py
@@ -351,8 +351,8 @@
"none": lambda optional, : None
}
-def simple_performance():
- env = celpy.Environment()
+def simple_performance(runner_class: type[celpy.Runner] | None = None) -> None:
+ env = celpy.Environment(runner_class=runner_class)
number = 100
compile = timeit.timeit(
@@ -364,8 +364,8 @@ def simple_performance():
'CEL_EXPRESSION_ORIGINAL_NO_OPTIONAL': CEL_EXPRESSION_ORIGINAL_NO_OPTIONAL
},
number=number
- ) / number
- print(f"Compile: {1_000 * compile:9.4f} ms")
+ )
+ print(f"Compile: {1_000 * compile / number:9.4f} ms")
ast = env.compile(CEL_EXPRESSION_ORIGINAL_NO_OPTIONAL)
@@ -380,8 +380,8 @@ def simple_performance():
'functions': functions
},
number=number
- ) / number
- print(f"Prepare: {1_000 * prepare:9.4f} ms")
+ )
+ print(f"Prepare: {1_000 * prepare / number:9.4f} ms")
program = env.program(ast, functions=functions)
@@ -398,8 +398,8 @@ def simple_performance():
"""),
globals={'celpy': celpy},
number=number
- ) / number
- print(f"Convert: {1_000 * convert:9.4f} ms")
+ )
+ print(f"Convert: {1_000 * convert / number:9.4f} ms")
cel_context = {
"class_a": celpy.json_to_cel({"property_a": "something"}),
@@ -419,8 +419,8 @@ def simple_performance():
'cel_context': cel_context
},
number=number
- ) / number
- print(f"Evaluate: {1_000 * evaluation:9.4f} ms")
+ )
+ print(f"Evaluate: {1_000 * evaluation / number:9.4f} ms")
print()
@@ -450,7 +450,18 @@ def detailed_profile():
ps.print_stats()
def main():
- simple_performance()
+ print("# Performance")
+ print()
+ print("## Interpreter")
+ print()
+ simple_performance(celpy.InterpretedRunner)
+ print()
+ print("## Transpiler")
+ print()
+ simple_performance(celpy.CompiledRunner)
+ print()
+ print("# Profile")
+ print()
detailed_profile()
if __name__ == "__main__":
diff --git a/docs/build/doctrees/api.doctree b/docs/build/doctrees/api.doctree
index d90706d..9a15a5c 100644
Binary files a/docs/build/doctrees/api.doctree and b/docs/build/doctrees/api.doctree differ
diff --git a/docs/build/doctrees/c7n_functions.doctree b/docs/build/doctrees/c7n_functions.doctree
index 18c60f8..d159268 100644
Binary files a/docs/build/doctrees/c7n_functions.doctree and b/docs/build/doctrees/c7n_functions.doctree differ
diff --git a/docs/build/doctrees/cli.doctree b/docs/build/doctrees/cli.doctree
index 2b65e8a..7892a28 100644
Binary files a/docs/build/doctrees/cli.doctree and b/docs/build/doctrees/cli.doctree differ
diff --git a/docs/build/doctrees/configuration.doctree b/docs/build/doctrees/configuration.doctree
index 83b4843..a82ec52 100644
Binary files a/docs/build/doctrees/configuration.doctree and b/docs/build/doctrees/configuration.doctree differ
diff --git a/docs/build/doctrees/environment.pickle b/docs/build/doctrees/environment.pickle
index 7600c53..dd0b2a0 100644
Binary files a/docs/build/doctrees/environment.pickle and b/docs/build/doctrees/environment.pickle differ
diff --git a/docs/build/doctrees/index.doctree b/docs/build/doctrees/index.doctree
index f78755c..0d1d704 100644
Binary files a/docs/build/doctrees/index.doctree and b/docs/build/doctrees/index.doctree differ
diff --git a/docs/build/doctrees/installation.doctree b/docs/build/doctrees/installation.doctree
index 11aa29a..dd68c27 100644
Binary files a/docs/build/doctrees/installation.doctree and b/docs/build/doctrees/installation.doctree differ
diff --git a/docs/build/doctrees/integration.doctree b/docs/build/doctrees/integration.doctree
index 4e5e354..de7c8e2 100644
Binary files a/docs/build/doctrees/integration.doctree and b/docs/build/doctrees/integration.doctree differ
diff --git a/docs/build/doctrees/structure.doctree b/docs/build/doctrees/structure.doctree
index 5c0cba0..ee659a0 100644
Binary files a/docs/build/doctrees/structure.doctree and b/docs/build/doctrees/structure.doctree differ
diff --git a/docs/build/html/.buildinfo b/docs/build/html/.buildinfo
index 21ac932..88859b4 100644
--- a/docs/build/html/.buildinfo
+++ b/docs/build/html/.buildinfo
@@ -1,4 +1,4 @@
# Sphinx build info version 1
# This file records the configuration used when building these files. When it is not found, a full rebuild will be done.
-config: 17638f2de9b0811238b70ffc830291fb
+config: 3f3e9d6f3a661b200fa79b5036f419ce
tags: 645f666f9bcd5a90fca523b33c5a78b7
diff --git a/docs/build/html/.buildinfo.bak b/docs/build/html/.buildinfo.bak
index ae47034..1791560 100644
--- a/docs/build/html/.buildinfo.bak
+++ b/docs/build/html/.buildinfo.bak
@@ -1,4 +1,4 @@
# Sphinx build info version 1
# This file records the configuration used when building these files. When it is not found, a full rebuild will be done.
-config: 5e7e78792cf4cfb4deae9f0ba2edb8f7
+config: 4e2cb0e805ddcba79237f83ebf4c7647
tags: 645f666f9bcd5a90fca523b33c5a78b7
diff --git a/docs/build/html/_images/plantuml-01f21512c6b94e19efccb4676b518bed63780ddd.png b/docs/build/html/_images/plantuml-01f21512c6b94e19efccb4676b518bed63780ddd.png
new file mode 100644
index 0000000..080aa3a
Binary files /dev/null and b/docs/build/html/_images/plantuml-01f21512c6b94e19efccb4676b518bed63780ddd.png differ
diff --git a/docs/build/html/_images/plantuml-165a23eb11f806bdd308ffac47a78125152ade58.png b/docs/build/html/_images/plantuml-165a23eb11f806bdd308ffac47a78125152ade58.png
new file mode 100644
index 0000000..b02b306
Binary files /dev/null and b/docs/build/html/_images/plantuml-165a23eb11f806bdd308ffac47a78125152ade58.png differ
diff --git a/docs/build/html/_images/plantuml-59894b154086c2a2ebe8ae46d280279ecce7cf8f.png b/docs/build/html/_images/plantuml-59894b154086c2a2ebe8ae46d280279ecce7cf8f.png
new file mode 100644
index 0000000..16f9a9c
Binary files /dev/null and b/docs/build/html/_images/plantuml-59894b154086c2a2ebe8ae46d280279ecce7cf8f.png differ
diff --git a/docs/build/html/_images/plantuml-60c95188891b5d1267db67bb61a17e9fd7c08060.png b/docs/build/html/_images/plantuml-60c95188891b5d1267db67bb61a17e9fd7c08060.png
new file mode 100644
index 0000000..cb501f6
Binary files /dev/null and b/docs/build/html/_images/plantuml-60c95188891b5d1267db67bb61a17e9fd7c08060.png differ
diff --git a/docs/build/html/_images/plantuml-a8a77cea7a1f2555dbb1cb6cea10dc21777c76bf.png b/docs/build/html/_images/plantuml-a8a77cea7a1f2555dbb1cb6cea10dc21777c76bf.png
new file mode 100644
index 0000000..96c3652
Binary files /dev/null and b/docs/build/html/_images/plantuml-a8a77cea7a1f2555dbb1cb6cea10dc21777c76bf.png differ
diff --git a/docs/build/html/_images/plantuml-ac649ac296fc6bea0cee4f6cb2d92533640e6763.png b/docs/build/html/_images/plantuml-ac649ac296fc6bea0cee4f6cb2d92533640e6763.png
new file mode 100644
index 0000000..391d74a
Binary files /dev/null and b/docs/build/html/_images/plantuml-ac649ac296fc6bea0cee4f6cb2d92533640e6763.png differ
diff --git a/docs/build/html/_images/plantuml-c970d97dc7e0a41eb7666fff5d466440fc8d67cf.png b/docs/build/html/_images/plantuml-c970d97dc7e0a41eb7666fff5d466440fc8d67cf.png
new file mode 100644
index 0000000..75c2ffd
Binary files /dev/null and b/docs/build/html/_images/plantuml-c970d97dc7e0a41eb7666fff5d466440fc8d67cf.png differ
diff --git a/docs/build/html/_modules/celpy/__init__.html b/docs/build/html/_modules/celpy/__init__.html
index 24e928f..75f87b7 100644
--- a/docs/build/html/_modules/celpy/__init__.html
+++ b/docs/build/html/_modules/celpy/__init__.html
@@ -47,91 +47,40 @@
Source code for celpy.__init__
# See the License for the specific language governing permissions and limitations under the License."""
-Pure Python implementation of CEL.
-
-.. todo:: Consolidate __init__ and parser into one module?
-
-Visible interface to CEL. This exposes the :py:class:`Environment`,
-the :py:class:`Evaluator` run-time, and the :py:mod:`celtypes` module
-with Python types wrapped to be CEL compatible.
-
-Example
-=======
-
-Here's an example with some details::
-
- >>> import celpy
-
- # A list of type names and class bindings used to create an environment.
- >>> types = []
- >>> env = celpy.Environment(types)
-
- # Parse the code to create the CEL AST.
- >>> ast = env.compile("355. / 113.")
-
- # Use the AST and any overriding functions to create an executable program.
- >>> functions = {}
- >>> prgm = env.program(ast, functions)
-
- # Variable bindings.
- >>> activation = {}
-
- # Final evaluation.
- >>> try:
- ... result = prgm.evaluate(activation)
- ... error = None
- ... except CELEvalError as ex:
- ... result = None
- ... error = ex.args[0]
-
- >>> result # doctest: +ELLIPSIS
- DoubleType(3.14159...)
-
-Another Example
-===============
-
-See https://github.com/google/cel-go/blob/master/examples/simple_test.go
-
-The model Go we're sticking close to::
-
- d := cel.Declarations(decls.NewVar("name", decls.String))
- env, err := cel.NewEnv(d)
- if err != nil {
- log.Fatalf("environment creation error: %v\\n", err)
- }
- ast, iss := env.Compile(`"Hello world! I'm " + name + "."`)
- // Check iss for compilation errors.
- if iss.Err() != nil {
- log.Fatalln(iss.Err())
- }
- prg, err := env.Program(ast)
- if err != nil {
- log.Fatalln(err)
- }
- out, _, err := prg.Eval(map[string]interface{}{
- "name": "CEL",
- })
- if err != nil {
- log.Fatalln(err)
- }
- fmt.Println(out)
- // Output:Hello world! I'm CEL.
-
-Here's the Pythonic approach, using concept patterned after the Go implementation::
-
- >>> from celpy import *
- >>> decls = {"name": celtypes.StringType}
- >>> env = Environment(annotations=decls)
- >>> ast = env.compile('"Hello world! I\\'m " + name + "."')
- >>> out = env.program(ast).evaluate({"name": "CEL"})
- >>> print(out)
- Hello world! I'm CEL.
+The pure Python implementation of the Common Expression Language, CEL.
+This module defines an interface to CEL for integration into other Python applications.
+This exposes the :py:class:`Environment` used to compile the source module,
+the :py:class:`Runner` used to evaluate the compiled code,
+and the :py:mod:`celpy.celtypes` module with Python types wrapped to be CEL compatible.
+
+The way these classes are used is as follows:
+
+.. uml::
+
+ @startuml
+ start
+ :Gather (or define) annotations;
+ :Create ""Environment"";
+ :Compile CEL;
+ :Create ""Runner"" with any extension functions;
+ :Evaluate the ""Runner"" with a ""Context"";
+ stop
+ @enduml
+
+The explicit decomposition into steps permits
+two extensions:
+
+1. Transforming the AST to introduce any optimizations.
+
+2. Saving the :py:class:`Runner` instance to reuse an expression with new inputs."""
+importabcimportjson# noqa: F401importloggingimportsys
+fromtextwrapimportindentfromtypingimportAny,Dict,Optional,Type,castimportlark
@@ -151,6 +100,8 @@
[docs]
-classRunner:
-"""Abstract runner.
+classRunner(abc.ABC):
+"""Abstract runner for a compiled CEL program.
- Given an AST, this can evaluate the AST in the context of a specific activation
- with any override function definitions.
+ The :py:class:`Environment` creates a :py:class:`Runner` to permit
+ saving a ready-tp-evaluate, compiled CEL expression.
+ A :py:class:`Runner` will evaluate the AST in the context of a specific activation
+ with the provided variable values.
- .. todo:: add type adapter and type provider registries.
+ A :py:class:`Runner` class provides
+ the ``tree_node_class`` attribute to define the type for tree nodes.
+ This class information is used by the :py:class:`Environment` to tailor the ``Lark`` instance created.
+ The class named by the ``tree_node_class`` can include specialized AST features
+ needed by a :py:class:`Runner` instance.
+
+ .. todo:: For a better fit with Go language expectations
+
+ Consider addoing type adapter and type provider registries.
+ This would permit distinct sources of protobuf message types. """
+ tree_node_class:type=lark.Tree
+
ast:lark.Tree,functions:Optional[Dict[str,CELFunction]]=None,)->None:
+"""
+ Initialize this ``Runner`` with a given AST.
+ The Runner will have annotations take from the :py:class:`Environment`,
+ plus any unique functions defined here.
+ """self.logger=logging.getLogger(f"celpy.{self.__class__.__name__}")self.environment=environmentself.ast=astself.functions=functions
[docs]
- defnew_activation(self,context:Context)->Activation:
+ defnew_activation(self)->Activation:"""
- Builds the working activation from the environmental defaults.
+ Builds a new, working :py:class:`Activation` using the :py:class:`Environment` as defaults.
+ A Context will later be layered onto this for evaluation. """
- returnself.environment.activation().nested_activation(vars=context)
[docs]
+ @abc.abstractmethoddefevaluate(self,activation:Context)->celpy.celtypes.Value:# pragma: no cover
- raiseNotImplementedError
+"""
+ Given variable definitions in the :py:class:`celpy.evaluation.Context`, evaluate the given AST and return the resulting value.
+
+ Generally, this should raise an :exc:`CELEvalError` for most kinds of ordinary problems.
+ It may raise an :exc:`CELUnsupportedError` for future features that aren't fully implemented.
+ Any Python exception reflects a serious problem.
+ """
+ ...
@@ -204,16 +193,7 @@
Source code for celpy.__init__
[docs]classInterpretedRunner(Runner):"""
- Pure AST expression evaluator. Uses :py:class:`evaluation.Evaluator` class.
-
- Given an AST, this evauates the AST in the context of a specific activation.
-
- The returned value will be a celtypes type.
-
- Generally, this should raise an :exc:`CELEvalError` for most kinds of ordinary problems.
- It may raise an :exc:`CELUnsupportedError` for future features.
-
- .. todo:: Refractor the Evaluator constructor from evaluation.
+ An **Adapter** for the :py:class:`celpy.evaluation.Evaluator` class. """
[docs]classCompiledRunner(Runner):"""
- Python compiled expression evaluator. Uses Python byte code and :py:func:`eval`.
+ An **Adapter** for the :py:class:`celpy.evaluation.Transpiler` class.
- Given an AST, this evaluates the AST in the context of a specific activation.
+ A :py:class:`celpy.evaluation.Transpiler` instance transforms the AST into Python.
+ It uses :py:func:`compile` to create a code object.
+ The final :py:meth:`evaluate` method uses :py:func:`exec` to evaluate the code object.
- Transform the AST into Python, uses :py:func:`compile` to create a code object.
- Uses :py:func:`eval` to evaluate.
+ Note, this requires the ``celpy.evaluation.TranspilerTree`` classes
+ instead of the default ``lark.Tree`` class. """
+ tree_node_class:type=TranspilerTree
+
[docs]
- defevaluate(self,activation:Context)->celpy.celtypes.Value:
- # eval() code object with activation as locals, and built-ins as gobals.
- returnsuper().evaluate(activation)
+ defevaluate(self,context:Context)->celpy.celtypes.Value:
+"""
+ Use :py:func:`exec` to execute the code object.
+ """
+ value=self.tp.evaluate(context)
+ returnvalue
-# TODO: Refactor classes into a separate "cel_protobuf" module.
-# TODO: Becomes cel_protobuf.Int32Value
+# TODO: Refactor this class into a separate "cel_protobuf" module.
+# TODO: Rename this type to ``cel_protobuf.Int32Value``
[docs]classInt32Value(celpy.celtypes.IntType):
+"""A wrapper for int32 values."""
+
-# The "well-known" types in a google.protobuf package.
-# We map these to CEl types instead of defining additional Protobuf Types.
+# The "well-known" types in a ``google.protobuf`` package.
+# We map these to CEL types instead of defining additional Protobuf Types.# This approach bypasses some of the range constraints that are part of these types.# It may also cause values to compare as equal when they were originally distinct types.googleapis={
@@ -309,15 +303,19 @@
Source code for celpy.__init__
[docs]classEnvironment:
-"""Compiles CEL text to create an Expression object.
+"""
+ Contains the current evaluation context.
+ The :py:meth:`Environment.compile` method
+ compiles CEL text to create an AST.
- From the Go implementation, there are things to work with the type annotations:
+ The :py:meth:`Environment.program` method
+ packages the AST into a program ready for evaluation.
- - type adapters registry make other native types available for CEL.
+ .. todo:: For a better fit with Go language expectations
- - type providers registry make ProtoBuf types available for CEL.
+ - A type adapters registry makes other native types available for CEL.
- .. todo:: Add adapter and provider registries to the Environment.
+ - A type providers registry make ProtoBuf types available for CEL. """
@@ -348,7 +346,7 @@
Source code for celpy.__init__
self.annotations:Dict[str,Annotation]=annotationsor{}self.logger.debug("Type Annotations %r",self.annotations)self.runner_class:Type[Runner]=runner_classorInterpretedRunner
- self.cel_parser=CELParser()
+ self.cel_parser=CELParser(tree_class=self.runner_class.tree_node_class)self.runnable:Runner# Fold in standard annotations. These (generally) define well-known protobuf types.
@@ -357,6 +355,12 @@
Source code for celpy.__init__
# We'd like to add 'type.googleapis.com/google' directly, but it seems to be an alias# for 'google', the path after the '/' in the uri.
+
defprogram(self,expr:lark.Tree,functions:Optional[Dict[str,CELFunction]]=None)->Runner:
-"""Transforms the AST into an executable runner."""
+"""
+ Transforms the AST into an executable :py:class:`Runner` object.
+
+ :param expr: The parse tree from :py:meth:`compile`.
+ :param functions: Any additional functions to be used by this CEL expression.
+ :returns: A :py:class:`Runner` instance that can be evaluated with a ``Context`` that provides values.
+ """self.logger.debug("Package %r",self.package)runner_class=self.runner_classself.runnable=runner_class(self,expr,functions)
+ self.logger.debug("Runnable %r",self.runnable)returnself.runnable
-
-
-
-[docs]
- defactivation(self)->Activation:
-"""Returns a base activation"""
- activation=Activation(package=self.package,annotations=self.annotations)
- returnactivation
# See the License for the specific language governing permissions and limitations under the License."""
-Pure Python implementation of CEL.
-
-This provides a few jq-like, bc-like, and shell expr-like features.
-
-- ``jq`` uses ``.`` to refer the current document. By setting a package
- name of ``"jq"`` and placing the JSON object in the package, we achieve
- similar syntax.
-
-- ``bc`` offers complex function definitions and other programming support.
- CEL can only evaluate a few bc-like expressions.
-
-- This does everything ``expr`` does, but the syntax is slightly different.
- The output of comparisons -- by default -- is boolean, where ``expr`` is an integer 1 or 0.
- Use ``-f 'd'`` to see decimal output instead of Boolean text values.
-
-- This does some of what ``test`` does, without a lot of the sophisticated
- file system data gathering.
- Use ``-b`` to set the exit status code from a Boolean result.
-
-TODO: This can also have a REPL, as well as process CSV files.
-
-SYNOPSIS
-========
-
-::
-
- python -m celpy [--arg name:type=value ...] [--null-input] expr
-
-Options:
-
-:--arg:
- Provides argument names, types and optional values.
- If the value is not provided, the name is expected to be an environment
- variable, and the value of the environment variable is converted and used.
-
-:--null-input:
- Normally, JSON documents are read from stdin in ndjson format. If no JSON documents are
- provided, the ``--null-input`` option skips trying to read from stdin.
-
-:expr:
- A CEL expression to evaluate.
-
-JSON documents are read from stdin in NDJSON format (http://jsonlines.org/, http://ndjson.org/).
-For each JSON document, the expression is evaluated with the document in a default
-package. This allows `.name` to pick items from the document.
-
-By default, the output is JSON serialized. This means strings will be JSON-ified and have quotes.
-
-If a ``--format`` option is provided, this is applied to the resulting object; this can be
-used to strip quotes, or limit precision on double objects, or convert numbers to hexadecimal.
-
-Arguments, Types, and Namespaces
-================================
-
-CEL objects rely on the celtypes definitions.
-
-Because of the close association between CEL and protobuf, some well-known protobuf types
-are also supported.
-
-.. todo:: CLI type environment
-
- Permit name.name:type=value to create namespace bindings.
-
-Further, type providers can be bound to CEL. This means an extended CEL
-may have additional types beyond those defined by the :py:class:`Activation` class.
+The CLI interface to ``celpy``.
+This parses the command-line options.
+It also offers an interactive REPL."""importargparseimportastimportcmd
+importdatetimeimportjsonimportloggingimportlogging.configimportosfrompathlibimportPathimportre
+importstatasos_statimportsys
-fromtypingimportAny,Callable,Dict,List,Optional,Tuple,cast
+fromtypingimportAny,Callable,Dict,List,Optional,Tuple,Union,casttry:importtomllib
@@ -223,7 +163,7 @@
ifoptions.arg:annotations={name:typeforname,type,valueinoptions.arg}else:
- annotations=None
+ annotations={}
+ annotations["stat"]=celtypes.FunctionType# If we're creating a named JSON document, we don't provide a default package.
- # If we're usinga JSON document to populate a package, we provide the given name.
+ # If we're using a JSON document to populate a package, we provide the given name.env=Environment(package=Noneifoptions.null_inputelseoptions.package,annotations=annotations,)try:expr=env.compile(options.expr)
- prgm=env.program(expr)
+ prgm=env.program(expr,functions={"stat":stat})exceptCELParseErrorasex:print(env.cel_parser.error_text(ex.args[0],ex.line,ex.column),file=sys.stderr
@@ -522,17 +534,19 @@
+# SPDX-Copyright: Copyright (c) Capital One Services, LLC
+# SPDX-License-Identifier: Apache-2.0
+# Copyright 2020 Capital One Services, LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and limitations under the License.
+
+"""
+Converts some Python-native types into CEL structures.
+
+Currently, atomic Python objects have direct use of types in :mod:`celpy.celtypes`.
+
+Non-atomic Python objects are characterized by JSON and Protobuf messages.
+This module has functions to convert JSON objects to CEL.
+
+A proper protobuf decoder is TBD.
+
+A more sophisticated type injection capability may be needed to permit
+additional types or extensions to :mod:`celpy.celtypes`.
+"""
+
+importbase64
+importdatetime
+importjson
+fromtypingimportAny,Dict,List,Union,cast
+
+fromcelpyimportceltypes
+
+JSON=Union[Dict[str,Any],List[Any],bool,float,int,str,None]
+
+
+
+[docs]
+classCELJSONEncoder(json.JSONEncoder):
+"""
+ An Encoder to export CEL objects as JSON text.
+
+ This is **not** a reversible transformation. Some things are coerced to strings
+ without any more detailed type marker.
+ Specifically timestamps, durations, and bytes.
+ """
+
+
+[docs]
+ @staticmethod
+ defto_python(
+ cel_object:celtypes.Value,
+ )->Union[celtypes.Value,List[Any],Dict[Any,Any],bool]:
+"""Recursive walk through the CEL object, replacing BoolType with native bool instances.
+ This lets the :py:mod:`json` module correctly represent the obects
+ with JSON ``true`` and ``false``.
+
+ This will also replace ListType and MapType with native ``list`` and ``dict``.
+ All other CEL objects will be left intact. This creates an intermediate hybrid
+ beast that's not quite a :py:class:`celtypes.Value` because a few things have been replaced.
+ """
+ ifisinstance(cel_object,celtypes.BoolType):
+ returnTrueifcel_objectelseFalse
+ elifisinstance(cel_object,celtypes.ListType):
+ return[CELJSONEncoder.to_python(item)foritemincel_object]
+ elifisinstance(cel_object,celtypes.MapType):
+ return{
+ CELJSONEncoder.to_python(key):CELJSONEncoder.to_python(value)
+ forkey,valueincel_object.items()
+ }
+ else:
+ returncel_object
+[docs]
+classCELJSONDecoder(json.JSONDecoder):
+"""
+ An Encoder to import CEL objects from JSON to the extent possible.
+
+ This does not handle non-JSON types in any form. Coercion from string
+ to TimestampType or DurationType or BytesType is handled by celtype
+ constructors.
+ """
+
+
+
+
+
+
+
+
+
\ No newline at end of file
diff --git a/docs/build/html/_modules/celpy/c7nlib.html b/docs/build/html/_modules/celpy/c7nlib.html
new file mode 100644
index 0000000..c18d332
--- /dev/null
+++ b/docs/build/html/_modules/celpy/c7nlib.html
@@ -0,0 +1,1988 @@
+
+
+
+
+
+
+ celpy.c7nlib — CEL in Python documentation
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
Source code for celpy.c7nlib
+# SPDX-Copyright: Copyright (c) Capital One Services, LLC
+# SPDX-License-Identifier: Apache-2.0
+# Copyright 2020 Capital One Services, LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and limitations under the License.
+
+"""
+Functions for C7N features when evaluating CEL expressions.
+
+These functions provide a mapping between C7N features and CEL.
+
+These functions are exposed by the global ``FUNCTIONS`` dictionary that is provided
+to the CEL evaluation run-time to provide necessary C7N features.
+
+The functions rely on implementation details in the ``CELFilter`` class.
+
+The API
+=======
+
+A C7N implementation can use CEL expressions and the :py:mod:`c7nlib` module as follows::
+
+ class CELFilter(c7n.filters.core.Filter):
+ decls = {
+ "resource": celpy.celtypes.MapType,
+ "now": celpy.celtypes.TimestampType,
+ "event": celpy.celtypes.MapType,
+ }
+ decls.update(celpy.c7nlib.DECLARATIONS)
+
+ def __init__(self, expr: str) -> None:
+ self.expr = expr
+
+ def validate(self) -> None:
+ cel_env = celpy.Environment(
+ annotations=self.decls,
+ runner_class=c7nlib.C7N_Interpreted_Runner)
+ cel_ast = cel_env.compile(self.expr)
+ self.pgm = cel_env.program(cel_ast, functions=celpy.c7nlib.FUNCTIONS)
+
+ def process(self,
+ resources: Iterable[celpy.celtypes.MapType]) -> Iterator[celpy.celtypes.MapType]:
+ now = datetime.datetime.utcnow()
+ for resource in resources:
+ with C7NContext(filter=the_filter):
+ cel_activation = {
+ "resource": celpy.json_to_cel(resource),
+ "now": celpy.celtypes.TimestampType(now),
+ }
+ if self.pgm.evaluate(cel_activation):
+ yield resource
+
+The :py:mod:`celpy.c7nlib` library of functions is bound into the CEL :py:class:`celpy.__init__.Runner` object that's built from the AST.
+
+Several variables will be required in the :py:class:`celpy.evaluation.Activation` for use by most CEL expressions
+that implement C7N filters:
+
+:resource:
+ A JSON document describing a cloud resource.
+
+:now:
+ The current timestamp.
+
+:event:
+ May be needed; it should be a JSON document describing an AWS CloudWatch event.
+
+
+The type: value Features
+========================
+
+The core value features of C7N require a number of CEL extensions.
+
+- :func:`glob(string, pattern)` uses Python fnmatch rules. This implements ``op: glob``.
+
+- :func:`difference(list, list)` creates intermediate sets and computes the difference
+ as a boolean value. Any difference is True. This implements ``op: difference``.
+
+- :func:`intersect(list, list)` creats intermediate sets and computes the intersection
+ as a boolean value. Any interection is True. This implements ``op: intersect``.
+
+- :func:`normalize(string)` supports normalized comparison between strings.
+ In this case, it means lower cased and trimmed. This implements ``value_type: normalize``.
+
+- :func:`net.cidr_contains` checks to see if a given CIDR block contains a specific
+ address. See https://www.openpolicyagent.org/docs/latest/policy-reference/#net.
+
+- :func:`net.cidr_size` extracts the prefix length of a parsed CIDR block.
+
+- :func:`version` uses ``disutils.version.LooseVersion`` to compare version strings.
+
+- :func:`resource_count` function. This is TBD.
+
+The type: value_from features
+==============================
+
+This relies on ``value_from()`` and ``jmes_path_map()`` functions
+
+In context, it looks like this::
+
+ value_from("s3://c7n-resources/exemptions.json", "json")
+ .jmes_path_map('exemptions.ec2.rehydration.["IamInstanceProfile.Arn"][].*[].*[]')
+ .contains(resource["IamInstanceProfile"]["Arn"])
+
+The ``value_from()`` function reads values from a given URI.
+
+- A full URI for an S3 bucket.
+
+- A full URI for a server that supports HTTPS GET requests.
+
+If a format is given, this is used, otherwise it's based on the
+suffix of the path.
+
+The ``jmes_path_map()`` function compiles and applies a JMESPath
+expression against each item in the collection to create a
+new collection. To an extent, this repeats functionality
+from the ``map()`` macro.
+
+Additional Functions
+====================
+
+A number of C7N subclasses of ``Filter`` provide additional features. There are
+at least 70-odd functions that are expressed or implied by these filters.
+
+Because the CEL expressions are always part of a ``CELFilter``, all of these
+additional C7N features need to be transformed into "mixins" that are implemented
+in two places. The function is part of the legacy subclass of ``Filter``,
+and the function is also part of ``CELFilter``.
+
+::
+
+ class InstanceImageMixin:
+ # from :py:class:`InstanceImageBase` refactoring
+ def get_instance_image(self):
+ pass
+
+ class RelatedResourceMixin:
+ # from :py:class:`RelatedResourceFilter` mixin
+ def get_related_ids(self):
+ pass
+
+ def get_related(self):
+ pass
+
+ class CredentialReportMixin:
+ # from :py:class:`c7n.resources.iam.CredentialReport` filter.
+ def get_credential_report(self):
+ pass
+
+ class ResourceKmsKeyAliasMixin:
+ # from :py:class:`c7n.resources.kms.ResourceKmsKeyAlias`
+ def get_matching_aliases(self, resource):
+ pass
+
+ class CrossAccountAccessMixin:
+ # from :py:class:`c7n.filters.iamaccessfilter.CrossAccountAccessFilter`
+ def get_accounts(self, resource):
+ pass
+ def get_vpcs(self, resource):
+ pass
+ def get_vpces(self, resource):
+ pass
+ def get_orgids(self, resource):
+ pass
+ # from :py:class:`c7n.resources.secretsmanager.CrossAccountAccessFilter`
+ def get_resource_policy(self, resource):
+ pass
+
+ class SNSCrossAccountMixin:
+ # from :py:class:`c7n.resources.sns.SNSCrossAccount`
+ def get_endpoints(self, resource):
+ pass
+ def get_protocols(self, resource):
+ pass
+
+ class ImagesUnusedMixin:
+ # from :py:class:`c7n.resources.ami.ImageUnusedFilter`
+ def _pull_ec2_images(self, resource):
+ pass
+ def _pull_asg_images(self, resource):
+ pass
+
+ class SnapshotUnusedMixin:
+ # from :py:class:`c7n.resources.ebs.SnapshotUnusedFilter`
+ def _pull_asg_snapshots(self, resource):
+ pass
+ def _pull_ami_snapshots(self, resource):
+ pass
+
+ class IamRoleUsageMixin:
+ # from :py:class:`c7n.resources.iam.IamRoleUsage`
+ def service_role_usage(self, resource):
+ pass
+ def instance_profile_usage(self, resource):
+ pass
+
+ class SGUsageMixin:
+ # from :py:class:`c7n.resources.vpc.SGUsage`
+ def scan_groups(self, resource):
+ pass
+
+ class IsShieldProtectedMixin:
+ # from :py:mod:`c7n.resources.shield`
+ def get_type_protections(self, resource):
+ pass
+
+ class ShieldEnabledMixin:
+ # from :py:class:`c7n.resources.account.ShieldEnabled`
+ def account_shield_subscriptions(self, resource):
+ pass
+
+ class CELFilter(
+ InstanceImageMixin, RelatedResourceMixin, CredentialReportMixin,
+ ResourceKmsKeyAliasMixin, CrossAccountAccessMixin, SNSCrossAccountMixin,
+ ImagesUnusedMixin, SnapshotUnusedMixin, IamRoleUsageMixin, SGUsageMixin,
+ Filter,
+ ):
+ '''Container for functions used by c7nlib to expose data to CEL'''
+ def __init__(self, data, manager) -> None:
+ super().__init__(data, manager)
+ assert data["type"].lower() == "cel"
+ self.expr = data["expr"]
+ self.parser = c7n.filters.offhours.ScheduleParser()
+
+ def validate(self):
+ pass # See above example
+
+ def process(self, resources):
+ pass # See above example
+
+This is not the complete list. See the ``tests/test_c7nlib.py`` for the ``celfilter_instance``
+fixture which contains **all** of the functions required.
+
+C7N Context Object
+==================
+
+A number of the functions require access to C7N features that are not simply part
+of the resource being filtered. There are two alternative ways to handle this dependency:
+
+- A global C7N context object that has the current ``CELFilter`` providing
+ access to C7N internals.
+
+- A ``C7N`` argument to the functions that need C7N access.
+ This would be provided in the activation context for CEL.
+
+To keep the library functions looking simple, the module global ``C7N`` is used.
+This avoids introducing a non-CEL parameter to the :py:mod:`celpy.c7nlib` functions.
+
+The ``C7N`` context object contains the following attributes:
+
+:filter:
+ The original C7N ``Filter`` object. This provides access to the
+ resource manager. It can be used to manage supplemental
+ queries using C7N caches and other resource management.
+
+This is set by the :py:class:`C7NContext` prior to CEL evaluation.
+
+Name Resolution
+===============
+
+Note that names are **not** resolved via a lookup in the program object,
+an instance of the :py:class:`celpy.Runner` class. To keep these functions
+simple, the runner is not part of the run-time, and name resolution
+will appear to be "hard-wrired" among these functions.
+
+This is rarely an issue, since most of these functions are independent.
+The :func:`value_from` function relies on :func:`text_from` and :func:`parse_text`.
+Changing either of these functions with an override won't modify the behavior
+of :func:`value_from`.
+"""
+
+importcsv
+importfnmatch
+importio
+importipaddress
+importjson
+importlogging
+importos.path
+importsys
+importurllib.request
+importzlib
+fromcontextlibimportclosing
+frompackaging.versionimportVersion
+fromtypesimportTracebackType
+fromtypingimportAny,Callable,Dict,Iterator,List,Optional,Type,Union,cast
+
+frompendulumimportparseasparse_date
+importjmespath# type: ignore [import-untyped]
+
+fromcelpyimportInterpretedRunner,celtypes
+fromcelpy.adapterimportjson_to_cel
+fromcelpy.evaluationimportAnnotation,Context,Evaluator
+
+logger=logging.getLogger(f"celpy.{__name__}")
+
+
+
+[docs]
+classC7NContext:
+"""
+ Saves current C7N filter for use by functions in this module.
+
+ This is essential for making C7N filter available to *some* of these functions.
+
+ ::
+
+ with C7NContext(filter):
+ cel_prgm.evaluate(cel_activation)
+ """
+
+
+
+
+
+# An object used for access to the C7N filter.
+# A module global makes the interface functions much simpler.
+# They can rely on `C7N.filter` providing the current `CELFilter` instance.
+C7N=cast("C7NContext",None)
+
+
+
+[docs]
+defkey(source:celtypes.ListType,target:celtypes.StringType)->celtypes.Value:
+"""
+ The C7N shorthand ``tag:Name`` doesn't translate well to CEL. It extracts a single value
+ from a sequence of objects with a ``{"Key": x, "Value": y}`` structure; specifically,
+ the value for ``y`` when ``x == "Name"``.
+
+ This function locate a particular "Key": target within a list of {"Key": x, "Value", y} items,
+ returning the y value if one is found, null otherwise.
+
+ In effect, the ``key()`` function::
+
+ resource["Tags"].key("Name")["Value"]
+
+ is somewhat like::
+
+ resource["Tags"].filter(x, x["Key"] == "Name")[0]
+
+ But the ``key()`` function doesn't raise an exception if the key is not found,
+ instead it returns None.
+
+ We might want to generalize this into a ``first()`` reduction macro.
+ ``resource["Tags"].first(x, x["Key"] == "Name" ? x["Value"] : null, null)``
+ This macro returns the first non-null value or the default (which can be ``null``.)
+ """
+ key=celtypes.StringType("Key")
+ value=celtypes.StringType("Value")
+ matches:Iterator[celtypes.Value]=(
+ item
+ foriteminsource
+ ifcast(celtypes.StringType,cast(celtypes.MapType,item).get(key))==target
+ )
+ try:
+ returncast(celtypes.MapType,next(matches)).get(value)
+ exceptStopIteration:
+ returnNone
+
+
+
+
+[docs]
+defglob(text:celtypes.StringType,pattern:celtypes.StringType)->celtypes.BoolType:
+"""Compare a string with a pattern.
+
+ While ``"*.py".glob(some_string)`` seems logical because the pattern the more persistent object,
+ this seems to cause confusion.
+
+ We use ``some_string.glob("*.py")`` to express a regex-like rule. This parallels the CEL
+ `.matches()` method.
+
+ We also support ``glob(some_string, "*.py")``.
+ """
+ returnceltypes.BoolType(fnmatch.fnmatch(text,pattern))
+
+
+
+
+[docs]
+defdifference(left:celtypes.ListType,right:celtypes.ListType)->celtypes.BoolType:
+"""
+ Compute the difference between two lists. This is ordered set difference: left - right.
+ It's true if the result is non-empty: there is an item in the left, not present in the right.
+ It's false if the result is empty: the lists are the same.
+ """
+ returnceltypes.BoolType(bool(set(left)-set(right)))
+
+
+
+
+[docs]
+defintersect(left:celtypes.ListType,right:celtypes.ListType)->celtypes.BoolType:
+"""
+ Compute the intersection between two lists.
+ It's true if the result is non-empty: there is an item in both lists.
+ It's false if the result is empty: there is no common item between the lists.
+ """
+ returnceltypes.BoolType(bool(set(left)&set(right)))
+[docs]
+defunique_size(collection:celtypes.ListType)->celtypes.IntType:
+"""
+ Unique size of a list
+ """
+ returnceltypes.IntType(len(set(collection)))
+
+
+
+
+[docs]
+classIPv4Network(ipaddress.IPv4Network):
+ # Override for net 2 net containment comparison
+
+
+
+ contains=__contains__
+
+ ifsys.version_info.major==3andsys.version_info.minor<=6:# pragma: no cover
+
+ @staticmethod
+ def_is_subnet_of(a,b):# type: ignore[no-untyped-def]
+ try:
+ # Always false if one is v4 and the other is v6.
+ ifa._version!=b._version:
+ raiseTypeError(f"{a} and {b} are not of the same version")
+ return(
+ b.network_address<=a.network_address
+ andb.broadcast_address>=a.broadcast_address
+ )
+ exceptAttributeError:
+ raiseTypeError(
+ f"Unable to test subnet containment between {a} and {b}"
+ )
+
+ defsupernet_of(self,other):# type: ignore[no-untyped-def]
+"""Return True if this network is a supernet of other."""
+ returnself._is_subnet_of(other,self)# type: ignore[no-untyped-call]
+[docs]
+classComparableVersion(Version):
+"""
+ The old LooseVersion could fail on comparing present strings, used
+ in the value as shorthand for certain options.
+
+ The new Version doesn't fail as easily.
+ """
+
+
+[docs]
+deftext_from(
+ url:celtypes.StringType,
+)->celtypes.Value:
+"""
+ Read raw text from a URL. This can be expanded to accept S3 or other URL's.
+ """
+ req=urllib.request.Request(url,headers={"Accept-Encoding":"gzip"})
+ raw_data:str
+ withclosing(urllib.request.urlopen(req))asresponse:
+ ifresponse.info().get("Content-Encoding")=="gzip":
+ raw_data=zlib.decompress(response.read(),zlib.MAX_WBITS|32).decode(
+ "utf8"
+ )
+ else:
+ raw_data=response.read().decode("utf-8")
+ returnceltypes.StringType(raw_data)
+[docs]
+defvalue_from(
+ url:celtypes.StringType,
+ format:Optional[celtypes.StringType]=None,
+)->celtypes.Value:
+"""
+ Read values from a URL.
+
+ First, do :func:`text_from` to read the source.
+ Then, do :func:`parse_text` to parse the source, if needed.
+
+ This makes the format optional, and deduces it from the URL's path information.
+
+ C7N will generally replace this with a function
+ that leverages a more sophisticated :class:`c7n.resolver.ValuesFrom`.
+ """
+ supported_formats=("json","ndjson","ldjson","jsonl","txt","csv","csv2dict")
+
+ # 1. get format either from arg or URL
+ ifnotformat:
+ _,suffix=os.path.splitext(url)
+ format=celtypes.StringType(suffix[1:])
+ ifformatnotinsupported_formats:
+ raiseValueError(f"Unsupported format: {format!r}")
+
+ # 2. read raw data
+ # Note this is directly bound to text_from() and does not go though the environment
+ # or other CEL indirection.
+ raw_data=cast(celtypes.StringType,text_from(url))
+
+ # 3. parse physical format (json, ldjson, ndjson, jsonl, txt, csv, csv2dict)
+ returnparse_text(raw_data,format)
+
+
+
+
+[docs]
+defjmes_path(
+ source_data:celtypes.Value,path_source:celtypes.StringType
+)->celtypes.Value:
+"""
+ Apply JMESPath to an object read from from a URL.
+ """
+ expression=jmespath.compile(path_source)
+ returnjson_to_cel(expression.search(source_data))
+
+
+
+
+[docs]
+defjmes_path_map(
+ source_data:celtypes.ListType,path_source:celtypes.StringType
+)->celtypes.ListType:
+"""
+ Apply JMESPath to a each object read from from a URL.
+ This is for ndjson, nljson and jsonl files.
+ """
+ expression=jmespath.compile(path_source)
+ returnceltypes.ListType(
+ [json_to_cel(expression.search(row))forrowinsource_data]
+ )
+
+
+
+
+[docs]
+defmarked_key(
+ source:celtypes.ListType,target:celtypes.StringType
+)->celtypes.Value:
+"""
+ Examines a list of {"Key": text, "Value": text} mappings
+ looking for the given Key value.
+
+ Parses a ``message:action@action_date`` value into a mapping
+ {"message": message, "action": action, "action_date": action_date}
+
+ If no Key or no Value or the Value isn't the right structure,
+ the result is a null.
+ """
+ value=key(source,target)
+ ifvalueisNone:
+ returnNone
+ try:
+ msg,tgt=cast(celtypes.StringType,value).rsplit(":",1)
+ action,action_date_str=tgt.strip().split("@",1)
+ exceptValueError:
+ returnNone
+ returnceltypes.MapType(
+ {
+ celtypes.StringType("message"):celtypes.StringType(msg),
+ celtypes.StringType("action"):celtypes.StringType(action),
+ celtypes.StringType("action_date"):celtypes.TimestampType(action_date_str),
+ }
+ )
+
+
+
+
+[docs]
+defimage(resource:celtypes.MapType)->celtypes.Value:
+"""
+ Reach into C7N to get the image details for this EC2 or ASG resource.
+
+ Minimally, the creation date is transformed into a CEL timestamp.
+ We may want to slightly generalize this to json_to_cell() the entire Image object.
+
+ The following may be usable, but it seems too complex:
+
+ ::
+
+ C7N.filter.prefetch_instance_images(C7N.policy.resources)
+ image = C7N.filter.get_instance_image(resource["ImageId"])
+ return json_to_cel(image)
+
+ .. todo:: Refactor C7N
+
+ Provide the :py:class:`InstanceImageBase` mixin in a :py:class:`CELFilter` class.
+ We want to have the image details in the new :py:class:`CELFilter` instance.
+ """
+
+ # Assuming the :py:class:`CELFilter` class has this method extracted from the legacy filter.
+ # Requies the policy already did this: C7N.filter.prefetch_instance_images([resource]) to
+ # populate cache.
+ image=C7N.filter.get_instance_image(resource)
+
+ ifimage:
+ creation_date=image["CreationDate"]
+ image_name=image["Name"]
+ else:
+ creation_date="2000-01-01T01:01:01.000Z"
+ image_name=""
+
+ returnjson_to_cel({"CreationDate":parse_date(creation_date),"Name":image_name})
+
+
+
+
+[docs]
+defget_raw_metrics(request:celtypes.MapType)->celtypes.Value:
+"""
+ Reach into C7N and make a statistics request using the current C7N filter object.
+
+ The ``request`` parameter is the request object that is passed through to AWS via
+ the current C7N filter's manager. The request is a Mapping with the following keys and values:
+
+ ::
+
+ get_raw_metrics({
+ "Namespace": "AWS/EC2",
+ "MetricName": "CPUUtilization",
+ "Dimensions": {"Name": "InstanceId", "Value": resource.InstanceId},
+ "Statistics": ["Average"],
+ "StartTime": now - duration("4d"),
+ "EndTime": now,
+ "Period": duration("86400s")
+ })
+
+ The request is passed through to AWS more-or-less directly. The result is a CEL
+ list of values for then requested statistic. A ``.map()`` macro
+ can be used to compute additional details. An ``.exists()`` macro can filter the
+ data to look for actionable values.
+
+ We would prefer to refactor C7N and implement this with code something like this:
+
+ ::
+
+ C7N.filter.prepare_query(C7N.policy.resources)
+ data = C7N.filter.get_resource_statistics(client, resource)
+ return json_to_cel(data)
+
+ .. todo:: Refactor C7N
+
+ Provide a :py:class:`MetricsAccess` mixin in a :py:class:`CELFilter` class.
+ We want to have the metrics processing in the new :py:class:`CELFilter` instance.
+
+ """
+ client=C7N.filter.manager.session_factory().client("cloudwatch")
+ data=client.get_metric_statistics(
+ Namespace=request["Namespace"],
+ MetricName=request["MetricName"],
+ Statistics=request["Statistics"],
+ StartTime=request["StartTime"],
+ EndTime=request["EndTime"],
+ Period=request["Period"],
+ Dimensions=request["Dimensions"],
+ )["Datapoints"]
+
+ returnjson_to_cel(data)
+
+
+
+
+[docs]
+defget_metrics(
+ resource:celtypes.MapType,request:celtypes.MapType
+)->celtypes.Value:
+"""
+ Reach into C7N and make a statistics request using the current C7N filter.
+
+ This builds a request object that is passed through to AWS via the :func:`get_raw_metrics`
+ function.
+
+ The ``request`` parameter is a Mapping with the following keys and values:
+
+ ::
+
+ resource.get_metrics({"MetricName": "CPUUtilization", "Statistic": "Average",
+ "StartTime": now - duration("4d"), "EndTime": now, "Period": duration("86400s")}
+ ).exists(m, m < 30)
+
+ The namespace is derived from the ``C7N.policy``. The dimensions are derived from
+ the ``C7N.fiter.model``.
+
+ .. todo:: Refactor C7N
+
+ Provide a :py:class:`MetricsAccess` mixin in a :py:class:`CELFilter` class.
+ We want to have the metrics processing in the new :py:class:`CELFilter` instance.
+
+ """
+ dimension=C7N.filter.manager.get_model().dimension
+ namespace=C7N.filter.manager.resource_type
+ # TODO: Varies by resource/policy type. Each policy's model may have different dimensions.
+ dimensions=[{"Name":dimension,"Value":resource.get(dimension)}]
+ raw_metrics=get_raw_metrics(
+ cast(
+ celtypes.MapType,
+ json_to_cel(
+ {
+ "Namespace":namespace,
+ "MetricName":request["MetricName"],
+ "Dimensions":dimensions,
+ "Statistics":[request["Statistic"]],
+ "StartTime":request["StartTime"],
+ "EndTime":request["EndTime"],
+ "Period":request["Period"],
+ }
+ ),
+ )
+ )
+ returnjson_to_cel(
+ [
+ cast(Dict[str,celtypes.Value],item).get(request["Statistic"])
+ foritemincast(List[celtypes.Value],raw_metrics)
+ ]
+ )
+
+
+
+
+[docs]
+defget_raw_health_events(request:celtypes.MapType)->celtypes.Value:
+"""
+ Reach into C7N and make a health-events request using the current C7N filter.
+
+ The ``request`` parameter is the filter object that is passed through to AWS via
+ the current C7N filter's manager. The request is a List of AWS health events.
+
+ ::
+
+ get_raw_health_events({
+ "services": ["ELASTICFILESYSTEM"],
+ "regions": ["us-east-1", "global"],
+ "eventStatusCodes": ['open', 'upcoming'],
+ })
+ """
+ client=C7N.filter.manager.session_factory().client(
+ "health",region_name="us-east-1"
+ )
+ data=client.describe_events(filter=request)["events"]
+ returnjson_to_cel(data)
+
+
+
+
+[docs]
+defget_health_events(
+ resource:celtypes.MapType,statuses:Optional[List[celtypes.Value]]=None
+)->celtypes.Value:
+"""
+ Reach into C7N and make a health-event request using the current C7N filter.
+
+ This builds a request object that is passed through to AWS via the :func:`get_raw_health_events`
+ function.
+
+ .. todo:: Handle optional list of event types.
+ """
+ ifnotstatuses:
+ statuses=[celtypes.StringType("open"),celtypes.StringType("upcoming")]
+ phd_svc_name_map={
+ "app-elb":"ELASTICLOADBALANCING",
+ "ebs":"EBS",
+ "efs":"ELASTICFILESYSTEM",
+ "elb":"ELASTICLOADBALANCING",
+ "emr":"ELASTICMAPREDUCE",
+ }
+ m=C7N.filter.manager
+ service=phd_svc_name_map.get(m.data["resource"],m.get_model().service.upper())
+ raw_events=get_raw_health_events(
+ cast(
+ celtypes.MapType,
+ json_to_cel(
+ {
+ "services":[service],
+ "regions":[m.config.region,"global"],
+ "eventStatusCodes":statuses,
+ }
+ ),
+ )
+ )
+ returnraw_events
+
+
+
+
+[docs]
+defget_related_ids(
+ resource:celtypes.MapType,
+)->celtypes.Value:
+"""
+ Reach into C7N and make a get_related_ids() request using the current C7N filter.
+
+ .. todo:: Refactor C7N
+
+ Provide the :py:class:`RelatedResourceFilter` mixin in a :py:class:`CELFilter` class.
+ We want to have the related id's details in the new :py:class:`CELFilter` instance.
+ """
+
+ # Assuming the :py:class:`CELFilter` class has this method extracted from the legacy filter.
+ related_ids=C7N.filter.get_related_ids(resource)
+ returnjson_to_cel(related_ids)
+
+
+
+
+[docs]
+defget_related_sgs(
+ resource:celtypes.MapType,
+)->celtypes.Value:
+"""
+ Reach into C7N and make a get_related_sgs() request using the current C7N filter.
+ """
+ security_groups=C7N.filter.get_related_sgs(resource)
+ returnjson_to_cel(security_groups)
+
+
+
+
+[docs]
+defget_related_subnets(
+ resource:celtypes.MapType,
+)->celtypes.Value:
+"""
+ Reach into C7N and make a get_related_subnets() request using the current C7N filter.
+ """
+ subnets=C7N.filter.get_related_subnets(resource)
+ returnjson_to_cel(subnets)
+
+
+
+
+[docs]
+defget_related_nat_gateways(
+ resource:celtypes.MapType,
+)->celtypes.Value:
+"""
+ Reach into C7N and make a get_related_nat_gateways() request using the current C7N filter.
+ """
+ nat_gateways=C7N.filter.get_related_nat_gateways(resource)
+ returnjson_to_cel(nat_gateways)
+
+
+
+
+[docs]
+defget_related_igws(
+ resource:celtypes.MapType,
+)->celtypes.Value:
+"""
+ Reach into C7N and make a get_related_igws() request using the current C7N filter.
+ """
+ igws=C7N.filter.get_related_igws(resource)
+ returnjson_to_cel(igws)
+
+
+
+
+[docs]
+defget_related_security_configs(
+ resource:celtypes.MapType,
+)->celtypes.Value:
+"""
+ Reach into C7N and make a get_related_security_configs() request using the current C7N filter.
+ """
+ security_configs=C7N.filter.get_related_security_configs(resource)
+ returnjson_to_cel(security_configs)
+
+
+
+
+[docs]
+defget_related_vpc(
+ resource:celtypes.MapType,
+)->celtypes.Value:
+"""
+ Reach into C7N and make a get_related_vpc() request using the current C7N filter.
+ """
+ vpc=C7N.filter.get_related_vpc(resource)
+ returnjson_to_cel(vpc)
+
+
+
+
+[docs]
+defget_related_kms_keys(
+ resource:celtypes.MapType,
+)->celtypes.Value:
+"""
+ Reach into C7N and make a get_related_kms_keys() request using the current C7N filter.
+ """
+ vpc=C7N.filter.get_related_kms_keys(resource)
+ returnjson_to_cel(vpc)
+
+
+
+
+[docs]
+defsecurity_group(
+ security_group_id:celtypes.MapType,
+)->celtypes.Value:
+"""
+ Reach into C7N and make a get_related() request using the current C7N filter to get
+ the security group.
+
+ .. todo:: Refactor C7N
+
+ Provide the :py:class:`RelatedResourceFilter` mixin in a :py:class:`CELFilter` class.
+ We want to have the related id's details in the new :py:class:`CELFilter` instance.
+ See :py:class:`VpcSecurityGroupFilter` subclass of :py:class:`RelatedResourceFilter`.
+ """
+
+ # Assuming the :py:class:`CELFilter` class has this method extracted from the legacy filter.
+ security_groups=C7N.filter.get_related([security_group_id])
+ returnjson_to_cel(security_groups)
+
+
+
+
+[docs]
+defsubnet(
+ subnet_id:celtypes.Value,
+)->celtypes.Value:
+"""
+ Reach into C7N and make a get_related() request using the current C7N filter to get
+ the subnet.
+
+ .. todo:: Refactor C7N
+
+ Provide the :py:class:`RelatedResourceFilter` mixin in a :py:class:`CELFilter` class.
+ We want to have the related id's details in the new :py:class:`CELFilter` instance.
+ See :py:class:`VpcSubnetFilter` subclass of :py:class:`RelatedResourceFilter`.
+ """
+ # Get related ID's first, then get items for the related ID's.
+ subnets=C7N.filter.get_related([subnet_id])
+ returnjson_to_cel(subnets)
+
+
+
+
+[docs]
+defflow_logs(
+ resource:celtypes.MapType,
+)->celtypes.Value:
+"""
+ Reach into C7N and locate the flow logs using the current C7N filter.
+
+ .. todo:: Refactor C7N
+
+ Provide a separate function to get the flow logs, separate from the
+ the filter processing.
+
+ .. todo:: Refactor :func:`c7nlib.flow_logs` -- it exposes too much implementation detail.
+
+ """
+ # TODO: Refactor into a function in ``CELFilter``. Should not be here.
+ client=C7N.filter.manager.session_factory().client("ec2")
+ logs=client.describe_flow_logs().get("FlowLogs",())
+ m=C7N.filter.manager.get_model()
+ resource_map:Dict[str,List[Dict[str,Any]]]={}
+ forflinlogs:
+ resource_map.setdefault(fl["ResourceId"],[]).append(fl)
+ ifresource.get(m.id)inresource_map:
+ flogs=resource_map[cast(str,resource.get(m.id))]
+ returnjson_to_cel(flogs)
+ returnjson_to_cel([])
+
+
+
+
+[docs]
+defvpc(
+ vpc_id:celtypes.Value,
+)->celtypes.Value:
+"""
+ Reach into C7N and make a ``get_related()`` request using the current C7N filter to get
+ the VPC details.
+
+ .. todo:: Refactor C7N
+
+ Provide the :py:class:`RelatedResourceFilter` mixin in a :py:class:`CELFilter` class.
+ We want to have the related id's details in the new :py:class:`CELFilter` instance.
+ See :py:class:`VpcFilter` subclass of :py:class:`RelatedResourceFilter`.
+ """
+ # Assuming the :py:class:`CELFilter` class has this method extracted from the legacy filter.
+ vpc=C7N.filter.get_related([vpc_id])
+ returnjson_to_cel(vpc)
+
+
+
+
+[docs]
+defsubst(
+ jmes_path:celtypes.StringType,
+)->celtypes.StringType:
+"""
+ Reach into C7N and build a set of substitutions to replace text in a JMES path.
+
+ This is based on how :py:class:`c7n.resolver.ValuesFrom` works. There are
+ two possible substitution values:
+
+ - account_id
+ - region
+
+ :param jmes_path: the source
+ :return: A JMES with values replaced.
+ """
+
+ config_args={
+ "account_id":C7N.filter.manager.config.account_id,
+ "region":C7N.filter.manager.config.region,
+ }
+ returnceltypes.StringType(jmes_path.format(**config_args))
+
+
+
+
+[docs]
+defcredentials(resource:celtypes.MapType)->celtypes.Value:
+"""
+ Reach into C7N and make a get_related() request using the current C7N filter to get
+ the IAM-role credential details.
+
+ See :py:class:`c7n.resources.iam.CredentialReport` filter.
+ The `get_credential_report()` function does what we need.
+
+ .. todo:: Refactor C7N
+
+ Refactor the :py:class:`c7n.resources.iam.CredentialReport` filter into a
+ ``CredentialReportMixin`` mixin to the :py:class:`CELFilter` class.
+ The ``get_credential_report()`` function does what we need.
+ """
+ returnjson_to_cel(C7N.filter.get_credential_report(resource))
+
+
+
+
+[docs]
+defkms_alias(
+ vpc_id:celtypes.Value,
+)->celtypes.Value:
+"""
+ Reach into C7N and make a get_matching_aliases() request using the current C7N filter to get
+ the alias.
+
+ See :py:class:`c7n.resources.kms.ResourceKmsKeyAlias`.
+ The `get_matching_aliases()` function does what we need.
+
+ .. todo:: Refactor C7N
+
+ Refactor the :py:class:`c7n.resources.kms.ResourceKmsKeyAlias` filter into a
+ ``ResourceKmsKeyAliasMixin`` mixin to the :py:class:`CELFilter` class.
+ The ``get_matching_aliases()`` dfunction does what we need.
+ """
+ returnjson_to_cel(C7N.filter.get_matching_aliases())
+
+
+
+
+[docs]
+defkms_key(
+ key_id:celtypes.Value,
+)->celtypes.Value:
+"""
+ Reach into C7N and make a ``get_related()`` request using the current C7N filter to get
+ the key. We're looking for the c7n.resources.kms.Key resource manager to get the related key.
+
+ .. todo:: Refactor C7N
+
+ Provide the :py:class:`RelatedResourceFilter` mixin in a :py:class:`CELFilter` class.
+ """
+ key=C7N.filter.get_related([key_id])
+ returnjson_to_cel(key)
+
+
+
+
+[docs]
+defresource_schedule(
+ tag_value:celtypes.Value,
+)->celtypes.Value:
+"""
+ Reach into C7N and use the the :py:class:`c7n.filters.offhours.ScheduleParser` class
+ to examine the tag's value, the current time, and return a True/False.
+ This parser is the `parser` value of the :py:class:`c7n.filters.offhours.Time` filter class.
+ Using the filter's `parser.parse(value)` provides needed structure.
+
+ The `filter.parser.parse(value)` will transform text of the Tag value
+ into a dictionary. This is further transformed to something we can use in CEL.
+
+ From this
+ ::
+
+ off=[(M-F,21),(U,18)];on=[(M-F,6),(U,10)];tz=pt
+
+ C7N ScheduleParser produces this
+ ::
+
+ {
+ off: [
+ { days: [1, 2, 3, 4, 5], hour: 21 },
+ { days: [0], hour: 18 }
+ ],
+ on: [
+ { days: [1, 2, 3, 4, 5], hour: 6 },
+ { days: [0], hour: 10 }
+ ],
+ tz: "pt"
+ }
+
+ For CEL, we need this
+ ::
+
+ {
+ off: [
+ { days: [1, 2, 3, 4, 5], hour: 21, tz: "pt" },
+ { days: [0], hour: 18, tz: "pt" }
+ ],
+ on: [
+ { days: [1, 2, 3, 4, 5], hour: 6, tz: "pt" },
+ { days: [0], hour: 10, tz: "pt" }
+ ],
+ }
+
+ This lets a CEL expression use
+ ::
+
+ key("maid_offhours").resource_schedule().off.exists(s,
+ now.getDayOfWeek(s.tz) in s.days && now.getHour(s.tz) == s.hour)
+ """
+ c7n_sched_doc=C7N.filter.parser.parse(tag_value)
+ tz=c7n_sched_doc.pop("tz","et")
+ cel_sched_doc={
+ state:[
+ {"days":time["days"],"hour":time["hour"],"tz":tz}fortimeintime_list
+ ]
+ forstate,time_listinc7n_sched_doc.items()
+ }
+ returnjson_to_cel(cel_sched_doc)
+
+
+
+
+[docs]
+defget_accounts(
+ resource:celtypes.MapType,
+)->celtypes.Value:
+"""
+ Reach into C7N filter and get accounts for a given resource.
+ Used by resources like AMI's, log-groups, ebs-snapshot, etc.
+
+ .. todo:: Refactor C7N
+
+ Provide the :py:class:`c7n.filters.iamaccessfilter.CrossAccountAccessFilter`
+ as a mixin to ``CELFilter``.
+ """
+ returnjson_to_cel(C7N.filter.get_accounts())
+
+
+
+
+[docs]
+defget_vpcs(
+ resource:celtypes.MapType,
+)->celtypes.Value:
+"""
+ Reach into C7N filter and get vpcs for a given resource.
+ Used by resources like AMI's, log-groups, ebs-snapshot, etc.
+
+ .. todo:: Refactor C7N
+
+ Provide the :py:class:`c7n.filters.iamaccessfilter.CrossAccountAccessFilter`
+ as a mixin to ``CELFilter``.
+ """
+ returnjson_to_cel(C7N.filter.get_vpcs())
+
+
+
+
+[docs]
+defget_vpces(
+ resource:celtypes.MapType,
+)->celtypes.Value:
+"""
+ Reach into C7N filter and get vpces for a given resource.
+ Used by resources like AMI's, log-groups, ebs-snapshot, etc.
+
+ .. todo:: Refactor C7N
+
+ Provide the :py:class:`c7n.filters.iamaccessfilter.CrossAccountAccessFilter`
+ as a mixin to ``CELFilter``.
+
+ """
+ returnjson_to_cel(C7N.filter.get_vpces())
+
+
+
+
+[docs]
+defget_orgids(
+ resource:celtypes.MapType,
+)->celtypes.Value:
+"""
+ Reach into C7N filter and get orgids for a given resource.
+ Used by resources like AMI's, log-groups, ebs-snapshot, etc.
+
+ .. todo:: Refactor C7N
+
+ Provide the :py:class:`c7n.filters.iamaccessfilter.CrossAccountAccessFilter`
+ as a mixin to ``CELFilter``.
+ """
+ returnjson_to_cel(C7N.filter.get_orgids())
+
+
+
+
+[docs]
+defget_endpoints(
+ resource:celtypes.MapType,
+)->celtypes.Value:
+"""For sns resources
+
+ .. todo:: Refactor C7N
+
+ Provide the :py:class:`c7n.filters.iamaccessfilter.CrossAccountAccessFilter`
+ as a mixin to ``CELFilter``.
+ """
+ returnjson_to_cel(C7N.filter.get_endpoints())
+[docs]
+defget_resource_policy(
+ resource:celtypes.MapType,
+)->celtypes.Value:
+"""
+ Reach into C7N filter and get the resource policy for a given resource.
+ Used by resources like AMI's, log-groups, ebs-snapshot, etc.
+
+ .. todo:: Refactor C7N
+ """
+ returnjson_to_cel(C7N.filter.get_resource_policy())
+
+
+
+
+[docs]
+defdescribe_subscription_filters(
+ resource:celtypes.MapType,
+)->celtypes.Value:
+"""
+ For log-groups resources.
+
+ .. todo:: Refactor C7N
+
+ this should be directly available in CELFilter.
+ """
+ client=C7N.filter.manager.session_factory().client("logs")
+ returnjson_to_cel(
+ C7N.filter.manager.retry(
+ client.describe_subscription_filters,logGroupName=resource["logGroupName"]
+ ).get("subscriptionFilters",())
+ )
+
+
+
+
+[docs]
+defdescribe_db_snapshot_attributes(
+ resource:celtypes.MapType,
+)->celtypes.Value:
+"""
+ For rds-snapshot and ebs-snapshot resources
+
+ .. todo:: Refactor C7N
+
+ this should be directly available in CELFilter.
+ """
+ client=C7N.filter.manager.session_factory().client("ec2")
+ returnjson_to_cel(
+ C7N.filter.manager.retry(
+ client.describe_snapshot_attribute,
+ SnapshotId=resource["SnapshotId"],
+ Attribute="createVolumePermission",
+ )
+ )
+
+
+
+
+[docs]
+defarn_split(arn:celtypes.StringType,field:celtypes.StringType)->celtypes.Value:
+"""
+ Parse an ARN, removing a partivular field.
+ The field name must one one of
+ "partition", "service", "region", "account-id", "resource-type", "resource-id"
+ In the case of a ``resource-type/resource-id`` path, this will be a "resource-id" value,
+ and there will be no "resource-type".
+
+ Examples formats
+
+ ``arn:partition:service:region:account-id:resource-id``
+
+ ``arn:partition:service:region:account-id:resource-type/resource-id``
+
+ ``arn:partition:service:region:account-id:resource-type:resource-id``
+ """
+ field_names={
+ len(names):names
+ fornamesin[
+ ("partition","service","region","account-id","resource-id"),
+ (
+ "partition",
+ "service",
+ "region",
+ "account-id",
+ "resource-type",
+ "resource-id",
+ ),
+ ]
+ }
+ prefix,*fields=arn.split(":")
+ ifprefix!="arn":
+ raiseValueError(f"Not an ARN: {arn}")
+ mapping=dict(zip(field_names[len(fields)],fields))
+ returnjson_to_cel(mapping[field])
+
+
+
+
+[docs]
+defall_images()->celtypes.Value:
+"""
+ Depends on :py:meth:`CELFilter._pull_ec2_images` and :py:meth:`CELFilter._pull_asg_images`
+
+ See :py:class:`c7n.resources.ami.ImageUnusedFilter`
+ """
+ returnjson_to_cel(
+ list(C7N.filter._pull_ec2_images()|C7N.filter._pull_asg_images())
+ )
+
+
+
+
+[docs]
+defall_snapshots()->celtypes.Value:
+"""
+ Depends on :py:meth:`CELFilter._pull_asg_snapshots`
+ and :py:meth:`CELFilter._pull_ami_snapshots`
+
+ See :py:class:`c7n.resources.ebs.SnapshotUnusedFilter`
+ """
+ returnjson_to_cel(
+ list(C7N.filter._pull_asg_snapshots()|C7N.filter._pull_ami_snapshots())
+ )
+[docs]
+defshield_protection(resource:celtypes.MapType)->celtypes.Value:
+"""
+ Depends on the :py:meth:`c7n.resources.shield.IsShieldProtected.process` method.
+ This needs to be refactored and renamed to avoid collisions with other ``process()`` variants.
+
+ Applies to most resource types.
+ """
+ client=C7N.filter.manager.session_factory().client(
+ "shield",region_name="us-east-1"
+ )
+ protections=C7N.filter.get_type_protections(
+ client,C7N.filter.manager.get_model()
+ )
+ protected_resources=[p["ResourceArn"]forpinprotections]
+ returnjson_to_cel(protected_resources)
+
+
+
+
+[docs]
+defshield_subscription(resource:celtypes.MapType)->celtypes.Value:
+"""
+ Depends on :py:meth:`c7n.resources.account.ShieldEnabled.process` method.
+ This needs to be refactored and renamed to avoid collisions with other ``process()`` variants.
+
+ Applies to account resources only.
+ """
+ subscriptions=C7N.filter.account_shield_subscriptions(resource)
+ returnjson_to_cel(subscriptions)
+
+
+
+
+[docs]
+defweb_acls(resource:celtypes.MapType)->celtypes.Value:
+"""
+ Depends on :py:meth:`c7n.resources.cloudfront.IsWafEnabled.process` method.
+ This needs to be refactored and renamed to avoid collisions with other ``process()`` variants.
+ """
+ wafs=C7N.filter.manager.get_resource_manager("waf").resources()
+ waf_name_id_map={w["Name"]:w["WebACLId"]forwinwafs}
+ returnjson_to_cel(waf_name_id_map)
+[docs]
+classC7N_Interpreted_Runner(InterpretedRunner):
+"""
+ Extends the Evaluation to introduce the C7N CELFilter instance into the evaluation.
+
+ The variable is global to allow the functions to have the simple-looking argument
+ values that CEL expects. This allows a function in this module to reach outside CEL for
+ access to C7N's caches.
+
+ .. todo: Refactor to be a mixin to the Runner class hierarchy.
+ """
+
+
+
+
+
+
+
+
+
\ No newline at end of file
diff --git a/docs/build/html/_modules/celpy/celparser.html b/docs/build/html/_modules/celpy/celparser.html
index 2a3511c..24431b9 100644
--- a/docs/build/html/_modules/celpy/celparser.html
+++ b/docs/build/html/_modules/celpy/celparser.html
@@ -47,19 +47,19 @@
Source code for celpy.celparser
# See the License for the specific language governing permissions and limitations under the License."""
-CEL Parser.
+A **Facade** around the CEL parser.
-See https://github.com/google/cel-spec/blob/master/doc/langdef.md
+The Parser is an instance of the :py:class:`lark.Lark` class.
-https://github.com/google/cel-cpp/blob/master/parser/Cel.g4
+The grammar is in the ``cel.lark`` file.
-https://github.com/google/cel-go/blob/master/parser/gen/CEL.g4
+For more information on CEL syntax, see the following:
-Builds a parser from the supplied cel.lark grammar.
+- https://github.com/google/cel-spec/blob/master/doc/langdef.md
-.. todo:: Consider embedding the ``cel.lark`` file as a triple-quoted literal.
+- https://github.com/google/cel-cpp/blob/master/parser/Cel.g4
- This means fixing a LOT of \\'s. But it also eliminates a data file from the installation.
+- https://github.com/google/cel-go/blob/master/parser/gen/CEL.g4Example::
@@ -106,6 +106,8 @@
Source code for celpy.celparser
[docs]classCELParseError(Exception):
+"""A syntax error in the CEL expression."""
+
[docs]classCELParser:
-"""Wrapper for the CEL parser and the syntax error messages."""
+"""
+ Creates a Lark parser with the required options.
+
+ .. important:: **Singleton**
+
+ There is one CEL_PARSER instance created by this class.
+ This is an optimization for environments like C7N where numerous
+ CEL expressions may parsed.
+
+ ::
+
+ CELParse.CEL_PARSER = None
+
+ Is required to create another parser instance.
+ This is commonly required in test environments.
+
+ This is also an **Adapter** for the CEL parser to provide pleasant
+ syntax error messages.
+ """CEL_PARSER:Optional[Lark]=None
# See the License for the specific language governing permissions and limitations under the License."""
-CEL Types: wrappers on Python types to provide CEL semantics.
+Provides wrappers over Python types to provide CEL semantics.This can be used by a Python module to work with CEL-friendly values and CEL results.
@@ -323,12 +323,19 @@
Source code for celpy.celtypes
>>> logical_condition( ... BoolType(False), StringType("Not This"), StringType("that")) StringType('that')
+
+ .. TODO:: Consider passing closures instead of Values.
+
+ The function can evaluate e().
+ If it's True, return x().
+ If it's False, return y().
+ Otherwise, it's a CELEvalError, which is the result """ifnotisinstance(e,BoolType):raiseTypeError(f"Unexpected {type(e)} ? {type(x)} : {type(y)}")
- result=xifeelsey
- logger.debug("logical_condition(%r, %r, %r) = %r",e,x,y,result)
- returnresult
""" Native Python has a left-to-right rule. CEL && is commutative with non-Boolean values, including error objects.
+
+ .. TODO:: Consider passing closures instead of Values.
+
+ The function can evaluate x().
+ If it's False, then return False.
+ Otherwise, it's True or a CELEvalError, return y(). """ifnotisinstance(x,BoolType)andnotisinstance(y,BoolType):raiseTypeError(f"{type(x)}{x!r} and {type(y)}{y!r}")
@@ -360,14 +373,17 @@
Source code for celpy.celtypes
[docs]deflogical_not(x:Value)->Value:"""
- Native python `not` isn't fully exposed for CEL types.
+ A function for native python `not`.
+
+ This could almost be `logical_or = evaluation.boolean(operator.not_)`,
+ but the definition would expose Python's notion of "truthiness", which isn't appropriate for CEL. """ifisinstance(x,BoolType):
- result=BoolType(notx)
+ result_value=BoolType(notx)else:raiseTypeError(f"not {type(x)}")
- logger.debug("logical_not(%r) = %r",x,result)
- returnresult
[docs]deflogical_or(x:Value,y:Value)->Value:"""
- Native Python has a left-to-right rule: (True or y) is True, (False or y) is y.
- CEL || is commutative with non-Boolean values, including errors.
+ Native Python has a left-to-right rule: ``(True or y)`` is True, ``(False or y)`` is y.
+ CEL ``||`` is commutative with non-Boolean values, including errors. ``(x || false)`` is ``x``, and ``(false || y)`` is ``y``. Example 1::
@@ -391,7 +407,13 @@
Source code for celpy.celtypes
is a "True"
- If the operand(s) are not BoolType, we'll create an TypeError that will become a CELEvalError.
+ If the operand(s) are not ``BoolType``, we'll create an ``TypeError`` that will become a ``CELEvalError``.
+
+ .. TODO:: Consider passing closures instead of Values.
+
+ The function can evaluate x().
+ If it's True, then return True.
+ Otherwise, it's False or a CELEvalError, return y(). """ifnotisinstance(x,BoolType)andnotisinstance(y,BoolType):raiseTypeError(f"{type(x)}{x!r} or {type(y)}{y!r}")
@@ -414,9 +436,9 @@
Source code for celpy.celtypes
[docs]classBoolType(int):"""
- Native Python permits unary operators on Booleans.
+ Native Python permits all unary operators to work on ``bool`` objects.
- For CEL, We need to prevent -false from working.
+ For CEL, we need to prevent the CEL expression ``-false`` from working. """
+[docs]
+ defget(self,key:Any,default:Optional[Any]=None)->Value:
+"""There is no default provision in CEL, that's a Python feature."""
+ ifnotMapType.valid_key_type(key):
+ raiseTypeError(f"unsupported key type: {type(key)}")
+ ifkeyinself:
+ returnsuper().get(key)
+ elifdefaultisnotNone:
+ returncast(Value,default)
+ else:
+ raiseKeyError(key)
@@ -1229,6 +1274,12 @@
Source code for celpy.celtypes
defvalid_key_type(key:Any)->bool:"""Valid CEL key types. Plus native str for tokens in the source when evaluating ``e.f``"""returnisinstance(key,(IntType,UintType,BoolType,StringType,str))
returntselifisinstance(source,str):
- # Use dateutil to try a variety of text formats.
+ # Use ``pendulum`` to try a variety of text formats.parsed_datetime=cast(datetime.datetime,pendulum.parse(source))returnsuper().__new__(cls,
@@ -1429,20 +1486,20 @@
@classmethoddeftz_name_lookup(cls,tz_name:str)->Optional[datetime.tzinfo]:"""
- The :py:func:`dateutil.tz.gettz` may be extended with additional aliases.
+ The ``pendulum`` parsing may be extended with additional aliases. .. TODO: Permit an extension into the timezone lookup. Tweak ``celpy.celtypes.TimestampType.TZ_ALIASES``.
@@ -1714,13 +1771,13 @@
Source code for celpy.celtypes
A duration + timestamp is not implemented by the timedelta superclass; it is handled by the datetime superclass that implementes timestamp + duration. """
- result=super().__add__(other)
- ifresult==NotImplemented:
- returncast(DurationType,result)
+ result_value=super().__add__(other)
+ ifresult_value==NotImplemented:
+ returncast(DurationType,result_value)# This is handled by TimestampType; this is here for completeness, but isn't used.
- ifisinstance(result,(datetime.datetime,TimestampType)):
- returnTimestampType(result)# pragma: no cover
- returnDurationType(result)
+ ifisinstance(result_value,(datetime.datetime,TimestampType)):
+ returnTimestampType(result_value)# pragma: no cover
+ returnDurationType(result_value)
@@ -1731,13 +1788,13 @@
Source code for celpy.celtypes
Most cases are handled by TimeStamp. """
- result=super().__radd__(other)
- ifresult==NotImplemented:
- returncast(DurationType,result)
+ result_value=super().__radd__(other)
+ ifresult_value==NotImplemented:
+ returncast(DurationType,result_value)# This is handled by TimestampType; this is here for completeness, but isn't used.
- ifisinstance(result,(datetime.datetime,TimestampType)):
- returnTimestampType(result)
- returnDurationType(result)
- # def get(self, field: Any, default: Optional[Value] = None) -> Value:
- # """
- # Alternative implementation with descent to locate a deeply-buried field.
- # It seemed like this was the defined behavior. It turns it, it isn't.
- # The code is here in case we're wrong and it really is the defined behavior.
- #
- # Note. There is no default provision in CEL.
- # """
- # if field in self:
- # return super().get(field)
- #
- # def descend(message: MessageType, field: Value) -> MessageType:
- # if field in message:
- # return message
- # for k in message.keys():
- # found = descend(message[k], field)
- # if found is not None:
- # return found
- # return None
- #
- # sub_message = descend(self, field)
- # if sub_message is None:
- # return default
- # return sub_message.get(field)
-
[docs]
-classTypeType:
+classTypeType(type):"""
- Annotation used to mark protobuf type objects.
- We map these to CELTypes so that type name testing works.
+ This is primarily used as a function to extract type from an object or a type.
+ For consistence, we define it as a type so other types can extend it. """
- type_name_mapping={
- "google.protobuf.Duration":DurationType,
- "google.protobuf.Timestamp":TimestampType,
- "google.protobuf.Int32Value":IntType,
- "google.protobuf.Int64Value":IntType,
- "google.protobuf.UInt32Value":UintType,
- "google.protobuf.UInt64Value":UintType,
- "google.protobuf.FloatValue":DoubleType,
- "google.protobuf.DoubleValue":DoubleType,
- "google.protobuf.Value":MessageType,
- "google.protubuf.Any":MessageType,# Weird.
- "google.protobuf.Any":MessageType,
- "list_type":ListType,
- "map_type":MapType,
- "map":MapType,
- "list":ListType,
- "string":StringType,
- "bytes":BytesType,
- "bool":BoolType,
- "int":IntType,
- "uint":UintType,
- "double":DoubleType,
- "null_type":type(None),
- "STRING":StringType,
- "BOOL":BoolType,
- "INT64":IntType,
- "UINT64":UintType,
- "INT32":IntType,
- "UINT32":UintType,
- "BYTES":BytesType,
- "DOUBLE":DoubleType,
- }
-
-
# See the License for the specific language governing permissions and limitations under the License."""
-CEL Interpreter using the AST directly.
+Evaluates CEL expressions given an AST.
+
+There are two implementations:
+
+- Evaluator -- interprets the AST directly.
+
+- Transpiler -- transpiles the AST to Python, compiles the Python to create a code object, and then uses :py:func:`exec` to evaluate the code object.The general idea is to map CEL operators to Python operators and push thereal work off to Python objects defined by the :py:mod:`celpy.celtypes` module.
-CEL operator "+" is implemented by "_+_" function. We map this to :py:func:`operator.add`.
-This will then look for `__add__()` methods in the various :py:class:`celpy.celtypes.CELType`
+CEL operator ``+`` is implemented by a ``"_+_"`` function.
+We map this name to :py:func:`operator.add`.
+This will then look for :py:meth:`__add__` methods in the various :py:mod:`celpy.celtypes`types.In order to deal gracefully with missing and incomplete data,
-exceptions are turned into first-class :py:class:`Result` objects.
+checked exceptions are used.
+A raised exception is turned into first-class :py:class:`celpy.celtypes.Result` object.They're not raised directly, but instead saved as part of the evaluation so thatshort-circuit operators can ignore the exceptions.This means that Python exceptions like :exc:`TypeError`, :exc:`IndexError`, and :exc:`KeyError`are caught and transformed into :exc:`CELEvalError` objects.
-The :py:class:`Resut` type hint is a union of the various values that are encountered
+The :py:class:`celpy.celtypes.Result` type hint is a union of the various values that are encounteredduring evaluation. It's a union of the :py:class:`celpy.celtypes.CELTypes` type and the:exc:`CELEvalError` exception... important:: Debugging
- If the os environment variable ``CEL_TRACE`` is set, then detailed tracing of methods is made available.
+ If the OS environment variable :envvar:`CEL_TRACE` is set, then detailed tracing of methods is made available. To see the trace, set the logging level for ``celpy.Evaluator`` to ``logging.DEBUG``."""
@@ -80,8 +88,10 @@
-# This annotation describes a union of types, functions, and function types.
+# An Annotation describes a union of types, functions, and function types.Annotation=Union[
- celpy.celtypes.CELType,
+ celpy.celtypes.TypeType,Callable[...,celpy.celtypes.Value],# Conversion functions and protobuf message type
@@ -148,12 +158,12 @@
Source code for celpy.evaluation
-[docs]
+[docs]classCELSyntaxError(Exception):"""CEL Syntax error -- the AST did not have the expected structure."""
-[docs]
+[docs]classCELEvalError(Exception):"""CEL evaluation problem. This can be saved as a temporary value for later use. This is politely ignored by logic operators to provide commutative short-circuit.
@@ -190,7 +200,7 @@
-# The interim results extend ``celtypes`` to include itermediate ``CELEvalError`` exception objects.
+# The interim results extend ``celtypes`` to include intermediate ``CELEvalError`` exception objects.# These can be deferred as part of commutative logical_and and logical_or operations.# It includes the responses to ``type()`` queries, also.Result=Union[
@@ -362,7 +372,7 @@
Wrap a function to transform native Python exceptions to CEL CELEvalError values. Any exception of the given class is replaced with the new CELEvalError object.
- :param new_text: Text of the exception, e.g., "divide by zero", "no such overload")
+ :param new_text: Text of the exception, e.g., "divide by zero", "no such overload", this is the return value if the :exc:`CELEvalError` becomes the result. :param exc_class: A Python exception class to match, e.g. ZeroDivisionError, or a sequence of exception classes (e.g. (ZeroDivisionError, ValueError))
@@ -406,12 +416,12 @@
Source code for celpy.evaluation
-[docs]
+[docs]defboolean(function:Callable[...,celpy.celtypes.Value],)->Callable[...,celpy.celtypes.BoolType]:"""
- Wraps boolean operators to create CEL BoolType results.
+ Wraps operators to create CEL BoolType results. :param function: One of the operator.lt, operator.gt, etc. comparison functions :return: Decorated function with type coercion.
@@ -421,17 +431,17 @@
-[docs]
+[docs]defoperator_in(item:Result,container:Result)->Result:""" CEL contains test; ignores type errors.
@@ -452,43 +462,44 @@
Source code for celpy.evaluation
would do it. But. It's not quite right for the job. There need to be three results, something :py:func:`filter` doesn't handle.
- These are the chocies:
+ These are the choices: - True. There was a item found. Exceptions may or may not have been found.
- - False. No item found AND no expceptions.
+ - False. No item found AND no exceptions. - CELEvalError. No item found AND at least one exception. To an extent this is a little like the ``exists()`` macro. We can think of ``container.contains(item)`` as ``container.exists(r, r == item)``.
- However, exists() tends to silence exceptions, where this can expost them.
+ However, exists() tends to silence exceptions, where this can expose them. .. todo:: This may be better done as ``reduce(logical_or, (item == c for c in container), BoolType(False))`` """
- result:Result=celpy.celtypes.BoolType(False)
+ result_value:Result=celpy.celtypes.BoolType(False)forcincast(Iterable[Result],container):try:ifc==item:returncelpy.celtypes.BoolType(True)exceptTypeErrorasex:logger.debug("operator_in(%s, %s) --> %s",item,container,ex)
- result=CELEvalError("no such overload",ex.__class__,ex.args)
- logger.debug("operator_in(%r, %r) = %r",item,container,result)
- returnresult
-[docs]
+[docs]deffunction_size(container:Result)->Result:"""
- The size() function applied to a Value. Delegate to Python's :py:func:`len`.
+ The size() function applied to a Value.
+ This is delegated to Python's :py:func:`len`.
- (string) -> int string length
- (bytes) -> int bytes length
- (list(A)) -> int list size
- (map(A, B)) -> int map size
+ size(string) -> int string length
+ size(bytes) -> int bytes length
+ size(list(A)) -> int list size
+ size(map(A, B)) -> int map size For other types, this will raise a Python :exc:`TypeError`. (This is captured and becomes an :exc:`CELEvalError` Result.)
@@ -499,14 +510,190 @@
+[docs]
+deffunction_contains(
+ container:Union[
+ celpy.celtypes.ListType,celpy.celtypes.MapType,celpy.celtypes.StringType
+ ],
+ item:Result,
+)->Result:
+"""
+ The contains() function applied to a Container and a Value.
+ THis is delegated to the `contains` method of a class.
+ """
+ returncelpy.celtypes.BoolType(container.contains(cast(celpy.celtypes.Value,item)))
# User-defined functions can override items in this mapping.
-base_functions:Mapping[str,CELFunction]={
+base_functions:dict[str,CELFunction]={"!_":celpy.celtypes.logical_not,"-_":operator.neg,"_+_":operator.add,
@@ -514,49 +701,42 @@
Source code for celpy.evaluation
"_*_":operator.mul,"_/_":operator.truediv,"_%_":operator.mod,
- "_<_":boolean(operator.lt),
- "_<=_":boolean(operator.le),
- "_>=_":boolean(operator.ge),
- "_>_":boolean(operator.gt),
- "_==_":boolean(operator.eq),
- "_!=_":boolean(operator.ne),
+ "_<_":bool_lt,
+ "_<=_":bool_le,
+ "_>_":bool_gt,
+ "_>=_":bool_ge,
+ "_==_":bool_eq,
+ "_!=_":bool_ne,"_in_":operator_in,"_||_":celpy.celtypes.logical_or,"_&&_":celpy.celtypes.logical_and,"_?_:_":celpy.celtypes.logical_condition,"_[_]":operator.getitem,
+ # The "methods" are actually named functions that can be overridden.
+ # The function version delegates to class methods.
+ # Yes, it's a bunch of indirection, but it permits simple overrides.
+ # A number of types support "size" and "contains": StringType, MapType, ListType
+ # This is generally made available via the _in_ operator."size":function_size,
- # StringType methods
- "endsWith":lambdas,text:celpy.celtypes.BoolType(s.endswith(text)),
- "startsWith":lambdas,text:celpy.celtypes.BoolType(s.startswith(text)),
+ "contains":function_contains,
+ # Universally available
+ "type":celpy.celtypes.TypeType,
+ # StringType methods, used by :py:meth:`Evaluator.method_eval`
+ "endsWith":function_endsWith,
+ "startsWith":function_startsWith,"matches":function_matches,
- "contains":lambdas,text:celpy.celtypes.BoolType(textins),# TimestampType methods. Type details are redundant, but required because of the lambdas
- "getDate":lambdats,tz_name=None:celpy.celtypes.IntType(ts.getDate(tz_name)),
- "getDayOfMonth":lambdats,tz_name=None:celpy.celtypes.IntType(
- ts.getDayOfMonth(tz_name)
- ),
- "getDayOfWeek":lambdats,tz_name=None:celpy.celtypes.IntType(
- ts.getDayOfWeek(tz_name)
- ),
- "getDayOfYear":lambdats,tz_name=None:celpy.celtypes.IntType(
- ts.getDayOfYear(tz_name)
- ),
- "getFullYear":lambdats,tz_name=None:celpy.celtypes.IntType(
- ts.getFullYear(tz_name)
- ),
- "getMonth":lambdats,tz_name=None:celpy.celtypes.IntType(ts.getMonth(tz_name)),
+ "getDate":function_getDate,
+ "getDayOfMonth":function_getDayOfMonth,
+ "getDayOfWeek":function_getDayOfWeek,
+ "getDayOfYear":function_getDayOfYear,
+ "getFullYear":function_getFullYear,
+ "getMonth":function_getMonth,# TimestampType and DurationType methods
- "getHours":lambdats,tz_name=None:celpy.celtypes.IntType(ts.getHours(tz_name)),
- "getMilliseconds":lambdats,tz_name=None:celpy.celtypes.IntType(
- ts.getMilliseconds(tz_name)
- ),
- "getMinutes":lambdats,tz_name=None:celpy.celtypes.IntType(
- ts.getMinutes(tz_name)
- ),
- "getSeconds":lambdats,tz_name=None:celpy.celtypes.IntType(
- ts.getSeconds(tz_name)
- ),
+ "getHours":function_getHours,
+ "getMilliseconds":function_getMilliseconds,
+ "getMinutes":function_getMinutes,
+ "getSeconds":function_getSeconds,# type conversion functions"bool":celpy.celtypes.BoolType,"bytes":celpy.celtypes.BytesType,
@@ -569,39 +749,39 @@
-[docs]
+[docs]classReferent:""" A Name can refer to any of the following things:
- - Annotations -- initially most names are these
- or a CELFunction that may implement a type.
+ - ``Annotation`` -- initially most names are these.
+ Must be provided as part of the initialization.
+
+ - ``CELFunction`` -- a Python function to implement a CEL function or method. Must be provided as part of the initialization.
+ The type conversion functions are names in a ``NameContainer``.
- - NameContainer -- some names are these. This is true
- when the name is *not* provided as part of the initialization because
+ - ``NameContainer`` -- some names are these.
+ This is true when the name is *not* provided as part of the initialization because we discovered the name during type or environment binding.
- - celpy.celtypes.Value -- many annotations also have values.
+ - ``celpy.celtypes.Value`` -- many annotations also have values. These are provided **after** Annotations, and require them.
- - CELEvalError -- This seems unlikely, but we include it because it's possible.
-
- - Functions -- All of the type conversion functions are names in a NameContainer.
+ - ``CELEvalError`` -- This seems unlikely, but we include it because it's possible.
- A name can be ambiguous and refer to both a nested ``NameContainer`` as well
- as a ``celpy.celtypes.Value`` (usually a MapType instance.)
+ A name can be ambiguous and refer to a nested ``NameContainer`` as well
+ as a ``celpy.celtypes.Value`` (usually a ``MapType`` instance.) Object ``b`` has two possible meanings:
- - ``b.c`` is a NameContainer for ``c``, a string.
+ - ``b`` is a ``NameContainer`` with ``c``, a string or some other object.
- - ``b`` is a mapping, and ``b.c`` is syntax sugar for ``b['c']``.
+ - ``b`` is a ``MapType`` or ``MessageType``, and ``b.c`` is syntax sugar for ``b['c']``. The "longest name" rule means that the useful value is the "c" object in the nested ``NameContainer``.
@@ -616,26 +796,25 @@
Source code for celpy.evaluation
>>> b.value == nc True
- In effect, this class is
- ::
+ .. note:: Future Design
+
+ A ``Referent`` is (almost) a ``tuple[Annotation, NameContainer | None, Value | NotSetSentinel]``.
+ The current implementation is stateful, because values are optional and may be added later.
+ The use of a special sentinel to indicate the value was not set is a little akward.
+ It's not really a 3-tuple, because NameContainers don't have values; they are a kind of value.
+ (``None`` is a valid value, and can't be used for this.)
- Referent = Union[
- Annotation,
- celpy.celtypes.Value,
- CELEvalError,
- CELFunction,
- ]
+ It may be slightly simpler to use a union of two types:
+ ``tuple[Annotation] | tuple[Annotation, NameContainer | Value]``.
+ One-tuples capture the Annotation for a name; two-tuples capture Annotation and Value (or subsidiary NameContainer). """
-[docs]
+[docs]def__init__(self,ref_to:Optional[Annotation]=None,
- # Union[
- # None, Annotation, celpy.celtypes.Value, CELEvalError,
- # CELFunction, 'NameContainer'
- # ] = None
+ # TODO: Add value here, also, as a handy short-cut to avoid the value setter.)->None:self.annotation:Optional[Annotation]=Noneself.container:Optional["NameContainer"]=None
@@ -647,13 +826,13 @@
Source code for celpy.evaluation
CELFunction,"NameContainer",]=None
- self._value_set=False
+ self._value_set=False# Should NOT be private.ifref_to:self.annotation=ref_to
-# A name resolution context is a mapping from an identifer to a Value or a ``NameContainer``.
+# A name resolution context is a mapping from an identifier to a Value or a ``NameContainer``.# This reflects some murkiness in the name resolution algorithm that needs to be cleaned up.
-Context=Mapping[str,Union[Result,"NameContainer"]]
+Context=Mapping[str,Union[Result,"NameContainer","CELFunction"]]# Copied from cel.lark
@@ -714,7 +908,7 @@
Source code for celpy.evaluation
-[docs]
+[docs]classNameContainer(Dict[str,Referent]):""" A namespace that fulfills the CEL name resolution requirement.
@@ -754,33 +948,33 @@
Source code for celpy.evaluation
The annotations (i.e. protobuf message types) have only a fairly remote annotation without a value.
- Structure.
+ .. rubric:: Structure
- A NameContainer is a mapping from names to Referents.
+ A ``NameContainer`` is a mapping from names to ``Referent`` instances.
- A Referent can be one of three things.
+ A `Referent` can be one of several things, including... - A NameContainer further down the path - An Annotation
- - An Annotation with a value.
+ - An Annotation and a value
+ - A CELFunction (In effect, an Annotation of CELFunction, and a value of the function implementation.)
- Loading Names.
+ .. rubric:: Life and Content
- There are several "phases" to building the chain of ``NameContainer`` instances.
+ There are two phases to building the chain of ``NameContainer`` instances. 1. The ``Activation`` creates the initial ``name : annotation`` bindings. Generally, the names are type names, like "int", bound to :py:class:`celtypes.IntType`. In some cases, the name is a future variable name, "resource", bound to :py:class:`celtypes.MapType`.
- 2. The ``Activation`` creates a second ``NameContainer`` that has variable names.
- This has a reference back to the parent to resolve names that are types.
+ 2. The ``Activation`` updates some variables to provide values.
- This involves decomposing the paths of names to make a tree of nested ``NameContainers``.
+ A name is decomposed into a path to make a tree of nested ``NameContainers``. Upper-level containers don't (necessarily) have types or values -- they're merely ``NameContainer`` along the path to the target names.
- Resolving Names.
+ .. rubric:: Resolving Names See https://github.com/google/cel-spec/blob/master/doc/langdef.md#name-resolution
@@ -825,7 +1019,7 @@
Source code for celpy.evaluation
The evaluator's :meth:`ident_value` method resolves the identifier into the ``Referent``.
- Acceptance Test Case
+ .. rubric:: Acceptance Test Cases We have two names
@@ -849,7 +1043,7 @@
:param names: A dictionary of {"name1.name1....": Referent, ...} items. """forname,refers_toinnames.items():
- self.logger.debug("load_annotations %r : %r",name,refers_to)
+ # self.logger.debug("load_annotations %r : %r", name, refers_to)ifnotself.extended_name_path.match(name):raiseValueError(f"Invalid name {name}")
@@ -900,11 +1094,11 @@
Source code for celpy.evaluation
-[docs]
+[docs]defload_values(self,values:Context)->None:
-"""Update annotations with actual values."""
+"""Update any annotations with actual values."""forname,refers_toinvalues.items():
- self.logger.debug("load_values %r : %r",name,refers_to)
+ # self.logger.debug("load_values %r : %r", name, refers_to)ifnotself.extended_name_path.match(name):raiseValueError(f"Invalid name {name}")
@@ -918,12 +1112,12 @@
Source code for celpy.evaluation
ifref.containerisNone:ref.container=NameContainer(parent=self.parent)context=ref.container
- context.setdefault(final,Referent())# No annotation.
+ context.setdefault(final,Referent())# No annotation previously present.context[final].value=refers_to
-[docs]
+[docs]classNotFound(Exception):""" Raised locally when a name is not found in the middle of package search.
@@ -934,70 +1128,111 @@
Source code for celpy.evaluation
-[docs]
+[docs]@staticmethod
- defdict_find_name(some_dict:Dict[str,Referent],path:List[str])->Result:
+ defdict_find_name(
+ some_dict:Union[Dict[str,Referent],Referent],path:Sequence[str]
+ )->Referent:"""
- Extension to navgiate into mappings, messages, and packages.
+ Recursive navigation into mappings, messages, and packages.
+ These are not NameContainers (or Activations).
- :param some_dict: An instance of a MapType, MessageType, or PackageType.
- :param path: names to follow into the structure.
+ :param some_dict: An instance of a ``MapType``, ``MessageType``, or ``PackageType``.
+ :param path: sequence of names to follow into the structure. :returns: Value found down inside the structure. """ifpath:head,*tail=pathtry:returnNameContainer.dict_find_name(
- cast(Dict[str,Referent],some_dict[head]),tail
+ cast(Dict[str,Referent],some_dict)[head],tail)exceptKeyError:
- NameContainer.logger.debug("%r not found in %s",head,some_dict.keys())
+ NameContainer.logger.debug(
+ "%r not found in %s",
+ head,
+ cast(Dict[str,Referent],some_dict).keys(),
+ )raiseNameContainer.NotFound(path)else:
- returncast(Result,some_dict)
+ # End of the path, we found it.
+ ifisinstance(some_dict,Referent):# pragma: no cover
+ # Seems unlikely, but, just to be sure...
+ returnsome_dict
+ referent=Referent(celpy.celtypes.MapType)
+ referent.value=cast(celpy.celtypes.MapType,some_dict)
+ returnreferent
-[docs]
- deffind_name(self,path:List[str])->Union["NameContainer",Result]:
+[docs]
+ deffind_name(self,path:List[str])->Referent:""" Find the name by searching down through nested packages or raise NotFound.
+ Returns the Value associated with this Name.
+
This is a kind of in-order tree walk of contained packages.
+
+ The collaborator must choose the annotation or the value from the Referent.
+
+ .. todo:: Refactored to return Referent.
+
+ The collaborator must handle two distinct errors:
+
+ 1. ``self[head]`` has a ``KeyError`` exception -- while not found on this path, the collaborator should keep searching.
+ Eventually it will raise a final ``KeyError`` that maps to a ``CELEvalError``
+ This should be exposed as f"no such member in mapping: {ex.args[0]!r}"
+
+ 2. ``self[head].value`` has no value and the ``Referent`` returned the annotation instead of the value.
+ In transpiled Python code, this **should** be exposed as f"undeclared reference to {ex.args[0]!r} (in container {ex.args[1]!r})" """
- ifpath:
- head,*tail=path
- try:
- sub_context=self[head].value
- exceptKeyError:
- self.logger.debug("%r not found in %s",head,self.keys())
- raiseNameContainer.NotFound(path)
- ifisinstance(sub_context,NameContainer):
- returnsub_context.find_name(tail)
- elifisinstance(
- sub_context,
- (
- celpy.celtypes.MessageType,
- celpy.celtypes.MapType,
- celpy.celtypes.PackageType,
- dict,
- ),
- ):
- # Out of defined NameContainers, moving into Values: Messages, Mappings or Packages
- # Make a fake Referent return value.
- item:Union["NameContainer",Result]=NameContainer.dict_find_name(
- cast(Dict[str,Referent],sub_context),tail
- )
- returnitem
- else:
- # Fully matched. No more Referents with NameContainers or Referents with Mappings.
- returncast(NameContainer,sub_context)
+ ifnotpath:
+ # Already fully matched. This ``NameContainer`` is what they were looking for.
+ referent=Referent()
+ referent.value=self
+ returnreferent
+
+ # Find the head of the path.
+ head,*tail=path
+ try:
+ sub_context=self[head]
+ exceptKeyError:
+ self.logger.debug("%r not found in %r",head,list(self.keys()))
+ raiseNameContainer.NotFound(path)
+ ifnottail:
+ # Found what they were looking for
+ returnsub_context
+
+ # There are several special cases for the continued search.
+ self.logger.debug("%r%r%r",head,tail,sub_context)
+
+ # We found a NameContainer, simple recursion will do.
+ item:Referent
+ ifsub_context.container:# isinstance(sub_context, NameContainer):
+ returnsub_context.container.find_name(tail)
+
+ # Uncommon case: value with no annotation, and the value is a Message, Mapping, or Package
+ elifsub_context._value_setandisinstance(
+ sub_context.value,
+ (
+ celpy.celtypes.MessageType,
+ celpy.celtypes.MapType,
+ celpy.celtypes.PackageType,
+ dict,
+ ),
+ ):
+ item=NameContainer.dict_find_name(
+ cast(Dict[str,Referent],sub_context.value),tail
+ )
+ returnitem
+
+ # A primitive type, but not at the end of the path. Ugn.else:
- # Fully matched. This NameContainer is what we were looking for.
- returnself
+ raiseTypeError(f"{sub_context!r} not a container")
-[docs]
+[docs]defparent_iter(self)->Iterator["NameContainer"]:"""Yield this NameContainer and all of its parents to create a flat list."""yieldself
@@ -1006,7 +1241,7 @@
Source code for celpy.evaluation
-[docs]
+[docs]defresolve_name(self,package:Optional[str],name:str)->Referent:""" Search with less and less package prefix until we find the thing.
@@ -1015,11 +1250,11 @@
Source code for celpy.evaluation
If a.b is a name to be resolved in the context of a protobuf declaration with scope A.B, then resolution is attempted, in order, as
- 1. A.B.a.b. (Search for "a" in paackage "A.B"; the ".b" is handled separately.)
+ 1. A.B.a.b. (Search for "a" in package "A.B"; the ".b" is handled separately.)
- 2. A.a.b. (Search for "a" in paackage "A"; the ".b" is handled separately.)
+ 2. A.a.b. (Search for "a" in package "A"; the ".b" is handled separately.)
- 3. (finally) a.b. (Search for "a" in paackage None; the ".b" is handled separately.)
+ 3. (finally) a.b. (Search for "a" in package None; the ".b" is handled separately.) To override this behavior, one can use .a.b; this name will only be attempted to be resolved in the root scope, i.e. as a.b.
@@ -1029,22 +1264,23 @@
Source code for celpy.evaluation
Given a target, search through this ``NameContainer`` and all parents in the :meth:`parent_iter` iterable. The first name we find in the parent sequence is the goal.
- This is because values are first, type annotations are laast.
+ This is because values are first, type annotations are last. If we can't find the identifier with given package target, truncate the package name from the end to create a new target and try again. This is a bottom-up look that favors the longest name.
- :param package: Prefix string "name.name.name"
+ :param package: Prefix string "path.path.path" :param name: The variable we're looking for
- :return: Name resolution as a Rereferent, often a value, but maybe a package or an
+ :return: Name resolution as a ``Rereferent``, often a value, but maybe a package or an annotation.
+ :raises KeyError: if the name cannot be found in this ``NameContainer`` """self.logger.debug("resolve_name(%r.%r) in %s, parent=%s",package,name,
- self.keys,
+ list(self.keys()),self.parent,)# Longest Name
@@ -1053,31 +1289,34 @@
Source code for celpy.evaluation
else:target=[""]# Pool of matches
- matches:List[Tuple[List[str],Union["NameContainer",Result]]]=[]
+ matches:List[Tuple[List[str],Referent]]=[]# Target has an extra item to make the len non-zero.whilenotmatchesandtarget:target=target[:-1]
- forpinself.parent_iter():
+ forncinself.parent_iter():try:package_ident:List[str]=target+[name]
- match:Union["NameContainer",Result]=p.find_name(package_ident)
- matches.append((package_ident,match))
+ # Find the Referent for this name.
+ ref_to=nc.find_name(package_ident)
+ matches.append((package_ident,ref_to))exceptNameContainer.NotFound:# No matches; move to the parent and try again.passself.logger.debug("resolve_name: target=%s+[%r], matches=%s",target,name,matches)
+ # NOTE: There are two separate kinds of failures: no name at all, and no value for the name.
+ # This is the no name at all. The collaborator may need the value or the annotation.ifnotmatches:raiseKeyError(name)
+ # Find the longest name match and return the Referent.# This feels hackish -- it should be the first referent value.
- # Find the longest name match.p
- path,value=max(matches,key=lambdapath_value:len(path_value[0]))
- returncast(Referent,value)
+[docs]
+ defget(# type: ignore[override]
+ self,name:str,default:Optional[Referent]=None
+ )->Union[
+ Annotation,celpy.celtypes.Value,CELEvalError,CELFunction,"NameContainer"
+ ]:
+"""
+ Used by transpiled code to get values from a NameContainer of Referents.
+
+ .. important:: This does not get a Referent, it gets a value.
+
+ .. todo:: This is a poorly-chosen name; a number of related types **all** need to have a get_value() method.
+ """
+ returnself.resolve_name(None,name).value
-[docs]
+[docs]classActivation:""" Namespace with variable bindings and type name ("annotation") bindings.
+ Additionally, the pool of functions and types are here, also. .. rubric:: Life and Content
@@ -1107,22 +1364,22 @@
Source code for celpy.evaluation
A nested Activation is created each time we evaluate a macro.
- An Activation contains a ``NameContainer`` instance to resolve identifers.
+ An Activation contains a ``NameContainer`` instance to resolve identifiers. (This may be a needless distinction and the two classes could, perhaps, be combined.)
- .. todo:: The environment's annotations are type names used for protobuf.
+ These names include variables as well as type names used for protobuf and the internal CEL ``type()`` function. .. rubric:: Chaining/Nesting Activations can form a chain so locals are checked first. Activations can nest via macro evaluation, creating transient local variables.
+ Consider this CEL macro expression: :: ``"[2, 4, 6].map(n, n / 2)"``
- means nested activations with ``n`` bound to 2, 4, and 6 respectively.
- The resulting objects then form a resulting list.
+ This works via a nested activation with ``n`` bound to 2, 4, and 6 respectively. This is used by an :py:class:`Evaluator` as follows::
@@ -1159,95 +1416,122 @@
Source code for celpy.evaluation
"namespace resolution should try to find the longest prefix for the evaluator." Most names start with ``IDENT``, but a primary can start with ``.``.
+ A leading ``.`` changes the search order from most local first to root first. """
-[docs]
+[docs]def__init__(self,
+ *,# Keyword only, too many things here.annotations:Optional[Mapping[str,Annotation]]=None,
- package:Optional[str]=None,vars:Optional[Context]=None,
- parent:Optional["Activation"]=None,
+ functions:Optional[Union[Mapping[str,CELFunction],list[CELFunction]]]=None,
+ package:Optional[str]=None,
+ based_on:Optional["Activation"]=None,)->None:""" Create an Activation.
- The annotations are loaded first. The variables are loaded second, and placed
- in front of the annotations in the chain of name resolutions. Values come before
- annotations.
+ The annotations are loaded first. The variables and their values are loaded second, and placed
+ in front of the annotations in the chain of name resolutions.
+
+ The Evaluator and the Transpiler use this to resolve identifiers into types, values, or functions.
- :param annotations: Variables and type annotations.
+ :keyword annotations: Variables and type annotations. Annotations are loaded first to serve as defaults to create a parent NameContainer.
- :param package: The package name to assume as a prefix for name resolution.
- :param vars: Variables and their values, loaded to update the NameContainer.
- :param parent: A parent activation in the case of macro evaluations.
+ :keyword vars: Variables and their values, loaded to update the NameContainer.
+ :keyword functions: functions and their implementation, loaded to update the NameContainer.
+ :keyword package: The package name to assume as a prefix for name resolution.
+ :keyword based_on: A foundational activation on which this is based. """logger.debug(
- "Activation(annotations=%r, package=%r, vars=%r, parent=%s)",
+ "Activation(annotations=%r, vars=%r, functions=%r, package=%r, based_on=%s)",annotations,
- package,vars,
- parent,
+ functions,
+ package,
+ based_on,)
- # Seed the annotation identifiers for this activation.
+ # Seed the annotations for identifiers in this activation.self.identifiers:NameContainer=NameContainer(
- parent=parent.identifiersifparentelseNone
+ parent=based_on.identifiersifbased_onelseNone)ifannotationsisnotNone:self.identifiers.load_annotations(annotations)
+ ifvarsisnotNone:
+ # Set values from a dictionary of names and values.
+ self.identifiers.load_values(vars)
- # The name of the run-time package -- an assumed prefix for name resolution
- self.package=package
-
- # Create a child NameContainer with variables (if any.)
- ifvarsisNone:
- pass
- elifisinstance(vars,Activation):# pragma: no cover
- # Deprecated legacy feature.
- raiseNotImplementedError("Use Activation.clone()")
-
+ # Update this NameContainer functions (if any.)
+ self.functions:collections.ChainMap[str,CELFunction]
+ ifisinstance(functions,Sequence):
+ local_functions:dict[str,CELFunction]={
+ f.__name__:fforfinfunctionsor[]
+ }
+ self.functions=collections.ChainMap(local_functions,base_functions)
+ # self.identifiers.load_values(local_functions)
+ elifisinstance(functions,Mapping):
+ self.functions=collections.ChainMap(
+ cast(dict[str,CELFunction],functions),base_functions
+ )
+ eliffunctionsisNone:
+ self.functions=collections.ChainMap(base_functions)else:
- # Set values from a dictionary of names and values.
- self.identifiers.load_values(vars)
+ raiseValueError("functions not a mapping or sequence")# pragma: no cover
+
+ # The name of the run-time package -- an assumed prefix for name resolution
+ self.package=package
-[docs]
+[docs]defclone(self)->"Activation":""" Create a clone of this activation with a deep copy of the identifiers. """
+ logger.debug("Cloning an Activation...")clone=Activation()
- clone.package=self.packageclone.identifiers=self.identifiers.clone()
+ clone.functions=self.functions.copy()
+ clone.package=self.package
+ logger.debug("clone: %r",self)returnclone
-[docs]
+[docs]defnested_activation(self,
- annotations:Optional[Mapping[str,Annotation]]=None,
+ annotations:Optional[Mapping[str,Annotation]]=None,# Remove this.vars:Optional[Context]=None,)->"Activation":"""
- Create a nested sub-Activation that chains to the current activation.
- The sub-activations don't have the same implied package context,
+ Create an Activation based on the current activation.
+ This new Activation will be seeded from the current activation's
+ ``NameContainer``.
- :param annotations: Variable type annotations
- :param vars: Variables with literals to be converted to the desired types.
- :return: An ``Activation`` that chains to this Activation.
+ :param annotations: Optional type definitions for the new local variables.
+ :param vars: Local variables to be added when creating this activation.
+ :return: A subsidiary ``Activation`` that chains to this Activation. """
- new=Activation(
- annotations=annotations,vars=vars,parent=self,package=self.package
+ logger.debug("Creating nested Activation...")
+ nested=Activation(
+ annotations=annotations,# Replace with self.annotations.
+ vars=vars,
+ functions=self.functions,
+ package=self.package,
+ based_on=self,)
- returnnew
+ logger.debug("nested: %r",self)
+ returnnested
-[docs]
- defresolve_variable(self,name:str)->Union[Result,NameContainer]:
+[docs]
+ defresolve_variable(
+ self,name:str
+ )->Union[celpy.celtypes.Value,CELFunction,NameContainer]:"""Find the object referred to by the name. An Activation usually has a chain of NameContainers to be searched.
@@ -1255,62 +1539,80 @@
Source code for celpy.evaluation
A variable can refer to an annotation and/or a value and/or a nested container. Most of the time, we want the `value` attribute of the Referent. This can be a Result (a Union[Value, CelType])
+
+ There's a subtle difference between a variable without an annotation,
+ and a variable with an annotation, but without a value.
+ """
+ # Will be a Referent. Get Value or Type -- interpreter works with either.
+ logger.debug("resolve_variable(%r)",name)
+ try:
+ referent=self.identifiers.resolve_name(self.package,name)
+ returncast(Union[Result,NameContainer],referent.value)
+ exceptKeyError:
+ returnself.functions[name]
+
+
+
+[docs]
+ defresolve_function(
+ self,name:str
+ )->Union[CELFunction,celpy.celtypes.TypeType]:
+"""A short-cut to find functions without looking at Variables first."""
+ logger.debug("resolve_function(%r)",name)
+ returnself.functions[name]
+
+
+
+[docs]
+ def__getattr__(
+ self,name:str
+ )->Union[celpy.celtypes.Value,CELFunction,NameContainer]:
+"""Handle ``activation.name`` in transpiled code (or ``activation.get('name')``).
+
+ If the name is not in the Activation with a value, a ``NameError`` exception must be raised.
+
+ Note that :py:meth:`Activation.resolve_variable` depends on :py:meth:`NameContainer.find_name`.
+ The :py:meth:`NameContainer.find_name` method **also** find the value.
+
+ This is -- perhaps -- less than optimal because it can mask the no value set case. """
- container_or_value=self.identifiers.resolve_name(self.package,str(name))
- returncast(Union[Result,NameContainer],container_or_value)
+ # Will be a Referent. Get Value if it was set or raise error if no value set.
+ try:
+ referent=self.identifiers.resolve_name(self.package,name)
+ logger.debug("get/__getattr__(%r) ==> %r",name,referent)
+ ifreferent._value_set:
+ returncast(Union[Result,NameContainer],referent.value)
+ else:
+ ifreferent.container:
+ returnreferent.container
+ elifreferent.annotation:
+ returncast(Union[Result,NameContainer],referent.annotation)
+ else:
+ raiseRuntimeError(f"Corrupt {self!r}")# pragma: no cover
+ exceptKeyError:
+ logger.debug("get/__getattr__(%r) fallback to functions",name)
+ returnself.functions[name]
-[docs]
-classFindIdent(lark.visitors.Visitor_Recursive):
-"""Locate the ident token at the bottom of an AST.
-
- This is needed to find the bind variable for macros.
-
- It works by doing a "visit" on the entire tree, but saving
- the details of the ``ident`` nodes only.
- """
-
-
-[docs]
+[docs]classEvaluator(lark.visitors.Interpreter[Result]):""" Evaluate an AST in the context of a specific Activation.
@@ -1348,45 +1650,48 @@
Source code for celpy.evaluation
An AST node must call ``self.visit_children(tree)`` explicitly to build the values for all the children of this node.
- Exceptions.
+ .. rubric:: Exceptions To handle ``2 / 0 || true``, the ``||``, ``&&``, and ``?:`` operators do not trivially evaluate and raise exceptions. They bottle up the exceptions and treat them as a kind of undecided value.
- Identifiers.
+ .. rubric:: Identifiers Identifiers have three meanings: - An object. This is either a variable provided in the activation or a function provided when building an execution. Objects also have type annotations.
- - A type annotation without an object, This is used to build protobuf messages.
+ - A type annotation without an object. This is used to build protobuf messages. - A macro name. The ``member_dot_arg`` construct may have a macro. Plus the ``ident_arg`` construct may also have a ``dyn()`` or ``has()`` macro. See below for more.
- Other than macros, a name maps to an ``Referent`` instance. This will have an
- annotation and -- perhaps -- an associated object.
+ Other than macros, a name maps to an ``Referent`` instance.
+ This will have an annotation and -- perhaps -- an associated object. Names have nested paths. ``a.b.c`` is a mapping, ``a``, that contains a mapping, ``b``, that contains ``c``.
- **MACROS ARE SPECIAL**.
+ .. important MACROS ARE SPECIAL
+
+ They aren't simple functions.
+
+ The macros do not **all** simply visit their children to perform evaluation.
- The macros do not **all** simply visit their children to perform evaluation. There are three cases: - ``dyn()`` does effectively nothing.
- It visits it's children, but also provides progressive type resolution
+ It visits its children, but also provides progressive type resolution through annotation of the AST. - ``has()`` attempts to visit the child and does a boolean transformation on the result. This is a macro because it doesn't raise an exception for a missing member item reference, but instead maps an exception to False.
- It doesn't return the value found, for a member item reference; instead, it maps
+ It doesn't return the value found for a member item reference; instead, it maps this to True. - The various ``member.macro()`` constructs do **NOT** visit children.
@@ -1403,12 +1708,12 @@
Source code for celpy.evaluation
logger=logging.getLogger("celpy.Evaluator")
-[docs]
+[docs]def__init__(self,ast:lark.Tree,activation:Activation,
- functions:Union[Sequence[CELFunction],Mapping[str,CELFunction],None]=None,
+ # functions: Union[Sequence[CELFunction], Mapping[str, CELFunction], None] = None, # Refactor into Activation)->None:""" Create an evaluator for an AST with specific variables and functions.
@@ -1416,45 +1721,38 @@
Source code for celpy.evaluation
:param ast: The AST to evaluate. :param activation: The variable bindings to use. :param functions: The functions to use. If nothing is supplied, the default
- global `base_functions` are used. Otherwise a ChainMap is created so
+ global `base_functions` are used. Otherwise, a ``ChainMap`` is created so these local functions override the base functions. """self.ast=astself.base_activation=activationself.activation=self.base_activation
- self.functions:Mapping[str,CELFunction]
- ifisinstance(functions,Sequence):
- local_functions={f.__name__:fforfinfunctionsor[]}
- self.functions=collections.ChainMap(local_functions,base_functions)# type: ignore [arg-type]
- elifisinstance(functions,Mapping):
- self.functions=collections.ChainMap(functions,base_functions)# type: ignore [arg-type]
- else:
- self.functions=base_functionsself.level=0
- self.logger.debug("activation: %r",self.activation)
- self.logger.debug("functions: %r",self.functions)
+ # self.logger.debug("functions: %r", self.functions) # Refactor ``self.functions`` into an Activation
-[docs]
+[docs]defsub_evaluator(self,ast:lark.Tree)->"Evaluator":""" Build an evaluator for a sub-expression in a macro.
+
:param ast: The AST for the expression in the macro. :return: A new `Evaluator` instance. """
- returnEvaluator(ast,activation=self.activation,functions=self.functions)
+ returnEvaluator(ast,activation=self.activation)
-[docs]
+[docs]defset_activation(self,values:Context)->"Evaluator":"""
- Chain a new activation using the given Context.
+ Create a new activation using the given Context. This is used for two things:
- 1. Bind external variables like command-line arguments or environment variables.
+ 1. Bind external variables. Examples are command-line arguments and environment variables. 2. Build local variable(s) for macro evaluation. """
@@ -1465,7 +1763,7 @@
Source code for celpy.evaluation
-[docs]
+[docs]defident_value(self,name:str,root_scope:bool=False)->Result_Function:"""Resolve names in the current activation. This includes variables, functions, the type registry for conversions,
@@ -1473,16 +1771,17 @@
Source code for celpy.evaluation
We may be limited to root scope, which prevents searching through alternative protobuf package definitions.
+ In principle, this changes the order of the search. """
- try:
- returncast(Result,self.activation.resolve_variable(name))
- exceptKeyError:
- returnself.functions[name]
+ # except KeyError:
+ # return self.functions[name] # Refactor ``self.functions`` into an Activation
-[docs]
- defevaluate(self)->celpy.celtypes.Value:
+[docs]
+ defevaluate(self,context:Optional[Context]=None)->celpy.celtypes.Value:""" Evaluate this AST and return the value or raise an exception.
@@ -1490,9 +1789,11 @@
Source code for celpy.evaluation
- External clients want the value or the exception.
- - Internally, we sometimes want to silence CELEvalError exceptions so that
+ - Internally, we sometimes want to silence ``CELEvalError`` exceptions so that we can apply short-circuit logic and choose a non-exceptional result. """
+ ifcontext:
+ self.set_activation(context)value=self.visit(self.ast)ifisinstance(value,CELEvalError):raisevalue
@@ -1500,17 +1801,17 @@
Source code for celpy.evaluation
-[docs]
+[docs]defvisit_children(self,tree:lark.Tree)->List[Result]:"""Extend the superclass to track nesting and current evaluation context."""self.level+=1
- result=super().visit_children(tree)
+ result_value=super().visit_children(tree)self.level-=1
- returnresult
Function evaluation. - Object creation and type conversions.
- - Other built-in functions like size()
+ - Other functions like ``size()`` or ``type()`` - Extension functions """function:CELFunctiontry:# TODO: Transitive Lookup of function in all parent activation contexts.
- function=self.functions[name_token.value]
+ # function = self.functions[name_token.value] # Refactor ``self.functions`` into an Activation
+ function=self.activation.resolve_function(name_token.value)exceptKeyErrorasex:err=(f"undeclared reference to '{name_token}' "
@@ -1556,7 +1858,7 @@
exprlist:Optional[Iterable[Result]]=None,)->Result:"""
- Method evaluation. While are (nominally) attached to an object, the only thing
- actually special is that the object is the first parameter to a function.
+ Method evaluation. While these are (nominally) attached to an object,
+ that would make overrides complicated.
+ Instead, these are functions (which can be overridden).
+ The object must the first parameter to a function. """function:CELFunctiontry:# TODO: Transitive Lookup of function in all parent activation contexts.
- function=self.functions[method_ident.value]
+ # function = self.functions[method_ident.value] # Refactor ``self.functions`` into an Activation
+ function=self.activation.resolve_function(method_ident.value)exceptKeyErrorasex:self.logger.debug("method_eval(%r, %r, %s) --> %r",object,method_ident,exprlist,ex)
- self.logger.debug("functions: %s",self.functions)
+ self.logger.debug(
+ "functions: %s",self.activation.functions
+ )# Refactor ``self.functions`` into an Activationerr=(f"undeclared reference to {method_ident.value!r} "f"(in activation '{self.activation}')"
@@ -1610,7 +1917,7 @@
Source code for celpy.evaluation
-[docs]
+[docs]defmacro_has_eval(self,exprlist:lark.Tree)->celpy.celtypes.BoolType:""" The has(e.f) macro.
@@ -1628,14 +1935,14 @@
Source code for celpy.evaluation
- If f is a repeated field or map field, has(e.f) indicates whether the field is non-empty.
- - If f is a singular or oneof field, has(e.f) indicates whether the field is set.
+ - If f is a singular or "oneof" field, has(e.f) indicates whether the field is set. 4. If e evaluates to a protocol buffers version 3 message and f is a defined field: - If f is a repeated field or map field, has(e.f) indicates whether the field is non-empty.
- - If f is a oneof or singular message field, has(e.f) indicates whether the field
+ - If f is a "oneof" field or singular message field, has(e.f) indicates whether the field is set. - If f is some other singular field, has(e.f) indicates whether the field's value
@@ -1650,7 +1957,7 @@
The default implementation short-circuits and can ignore a CELEvalError in the two alternative sub-expressions.
- The conditional sub-expression CELEvalError is propogated out as the result.
+ The conditional sub-expression CELEvalError is propagated out as the result. See https://github.com/google/cel-spec/blob/master/doc/langdef.md#logical-operators
@@ -1673,7 +1980,8 @@
Source code for celpy.evaluation
returnvalues[0]eliflen(tree.children)==3:# full conditionalor "?" conditionalor ":" expr.
- func=self.functions["_?_:_"]
+ # func = self.functions["_?_:_"] # Refactor ``self.functions`` into an Activation
+ func=self.activation.resolve_function("_?_:_")cond_value=self.visit(cast(lark.Tree,tree.children[0]))left=right=cast(Result,celpy.celtypes.BoolType(False))try:
@@ -1700,7 +2008,7 @@
values = self.visit_children(tree) func = functions[op_name_map[tree.data]]
- result = func(*values)
+ result_value = func(*values) The AST doesn't provide a flat list of values, however. """
@@ -1804,6 +2114,7 @@
Source code for celpy.evaluation
eliflen(tree.children)==2:left_op,right_tree=cast(Tuple[lark.Tree,lark.Tree],tree.children)
+ # Map a node data in parse tree to an operation function.op_name={"relation_lt":"_<_","relation_le":"_<=_",
@@ -1813,7 +2124,8 @@
Source code for celpy.evaluation
"relation_ne":"_!=_","relation_in":"_in_",}[left_op.data]
- func=self.functions[op_name]
+ # func = self.functions[op_name] # Refactor ``self.functions`` into an Activation
+ func=self.activation.resolve_function(op_name)# NOTE: values have the structure [[left], right](left,*_),right=cast(Tuple[List[Result],Result],self.visit_children(tree)
@@ -1840,7 +2152,7 @@
values = self.visit_children(tree) func = functions[op_name_map[tree.data]]
- result = func(*values)
+ result_value = func(*values) The AST doesn't provide a flat list of values, however. """
@@ -1866,11 +2178,13 @@
Source code for celpy.evaluation
eliflen(tree.children)==2:left_op,right_tree=cast(Tuple[lark.Tree,lark.Tree],tree.children)
+ # Map a node data in parse tree to an operation function.op_name={"addition_add":"_+_","addition_sub":"_-_",}[left_op.data]
- func=self.functions[op_name]
+ # func = self.functions[op_name] # Refactor ``self.functions`` into an Activation
+ func=self.activation.resolve_function(op_name)# NOTE: values have the structure [[left], right](left,*_),right=cast(Tuple[List[Result],Result],self.visit_children(tree)
@@ -1904,7 +2218,7 @@
values = self.visit_children(tree) func = functions[op_name_map[tree.data]]
- result = func(*values)
+ result_value = func(*values) The AST doesn't provide a flat list of values, however. """
@@ -1931,12 +2245,14 @@
Source code for celpy.evaluation
eliflen(tree.children)==2:left_op,right_tree=cast(Tuple[lark.Tree,lark.Tree],tree.children)
+ # Map a node data in parse tree to an operation function.op_name={"multiplication_div":"_/_","multiplication_mul":"_*_","multiplication_mod":"_%_",}[left_op.data]
- func=self.functions[op_name]
+ # func = self.functions[op_name] # Refactor ``self.functions`` into an Activation
+ func=self.activation.resolve_function(op_name)# NOTE: values have the structure [[left], right](left,*_),right=cast(Tuple[List[Result],Result],self.visit_children(tree)
@@ -1977,7 +2293,7 @@
values = self.visit_children(tree) func = functions[op_name_map[tree.data]]
- result = func(*values)
+ result_value = func(*values) But, values has the structure ``[[], right]`` """iflen(tree.children)==1:
- # member with no preceeding unary_not or unary_neg
+ # member with no preceding unary_not or unary_neg# TODO: If there are two possible values (namespace v. mapping) chose the namespace.values=self.visit_children(tree)returnvalues[0]eliflen(tree.children)==2:op_tree,right_tree=cast(Tuple[lark.Tree,lark.Tree],tree.children)
+ # Map a node data in parse tree to an operation function.op_name={"unary_not":"!_","unary_neg":"-_",}[op_tree.data]
- func=self.functions[op_name]
+ # func = self.functions[op_name] # Refactor ``self.functions`` into an Activation
+ func=self.activation.resolve_function(op_name)# NOTE: values has the structure [[], right]left,right=cast(Tuple[List[Result],Result],self.visit_children(tree))self.logger.debug("unary %s%r",op_name,right)
@@ -2040,7 +2358,7 @@
"""args=cast(lark.Tree,child.children[2])var_tree,expr_tree=cast(Tuple[lark.Tree,lark.Tree],args.children)
- identifier=FindIdent.in_tree(var_tree)
- ifidentifierisNone:# pragma: no cover
- # This seems almost impossible.
- raiseCELSyntaxError(
+ idents=list(var_tree.find_data("ident"))
+ iflen(idents)!=1:
+ # Essentially impossible.
+ raiseCELSyntaxError(# pragma: no coverf"{child.data}{child.children}: bad macro node",line=child.meta.line,column=child.meta.column,)
+ identifier=cast(lark.Token,idents[0].children[0]).value
+ # identifier = FindIdent.in_tree(var_tree)
+ # if identifier is None: # pragma: no cover
+ # # This seems almost impossible.
+ # raise CELSyntaxError(
+ # f"{child.data} {child.children}: bad macro node",
+ # line=child.meta.line,
+ # column=child.meta.column,
+ # )# nested_eval = Evaluator(ast=expr_tree, activation=self.activation)nested_eval=self.sub_evaluator(ast=expr_tree)defsub_expr(v:celpy.celtypes.Value)->Any:try:
- returnnested_eval.set_activation({identifier:v}).evaluate()
+ returnnested_eval.evaluate({identifier:v})exceptCELEvalErrorasex:returnex
@@ -2122,12 +2449,12 @@
Source code for celpy.evaluation
-[docs]
+[docs]defbuild_reduce_macro_eval(self,child:lark.Tree)->Tuple[Callable[[Result,Result],Result],lark.Tree]:"""
- Builds macro function and intiial expression for reduce().
+ Builds macro function and initial expression for reduce(). For example
@@ -2148,28 +2475,37 @@
Source code for celpy.evaluation
reduce_var_tree,iter_var_tree,init_expr_tree,expr_tree=cast(Tuple[lark.Tree,lark.Tree,lark.Tree,lark.Tree],args.children)
- reduce_ident=FindIdent.in_tree(reduce_var_tree)
- iter_ident=FindIdent.in_tree(iter_var_tree)
- ifreduce_identisNoneoriter_identisNone:# pragma: no cover
+ reduce_idents=list(reduce_var_tree.find_data("ident"))
+ iter_idents=list(iter_var_tree.find_data("ident"))
+ iflen(reduce_idents)!=1orlen(iter_idents)!=1:# pragma: no cover# This seems almost impossible.raiseCELSyntaxError(f"{child.data}{child.children}: bad macro node",line=child.meta.line,column=child.meta.column,)
+ reduce_ident=cast(lark.Token,reduce_idents[0].children[0]).value
+ iter_ident=cast(lark.Token,iter_idents[0].children[0]).value
+ # reduce_ident = FindIdent.in_tree(reduce_var_tree)
+ # iter_ident = FindIdent.in_tree(iter_var_tree)
+ # if reduce_ident is None or iter_ident is None: # pragma: no cover
+ # # This seems almost impossible.
+ # raise CELSyntaxError(
+ # f"{child.data} {child.children}: bad macro node",
+ # line=child.meta.line,
+ # column=child.meta.column,
+ # )# nested_eval = Evaluator(ast=expr_tree, activation=self.activation)nested_eval=self.sub_evaluator(ast=expr_tree)defsub_expr(r:Result,i:Result)->Result:
- returnnested_eval.set_activation(
- {reduce_ident:r,iter_ident:i}
- ).evaluate()
+ returnnested_eval.evaluate({reduce_ident:r,iter_ident:i})returnsub_expr,init_expr_tree
- In all other cases, e.f evaluates to an error.
- TODO: implement member "." IDENT for messages.
+ TODO: implement member "." IDENT for protobuf message types. """member_tree,property_name_token=cast(Tuple[lark.Tree,lark.Token],tree.children)member=self.visit(member_tree)property_name=property_name_token.value
- result:Result
+ result_value:Resultifisinstance(member,CELEvalError):
- result=cast(Result,member)
+ result_value=cast(Result,member)elifisinstance(member,NameContainer):# Navigation through names provided as external run-time bindings.# The dict is the value of a Referent that was part of a namespace path.ifproperty_nameinmember:
- result=cast(Result,member[property_name].value)
+ result_value=cast(Result,member[property_name].value)else:err=f"No {property_name!r} in bindings {sorted(member.keys())}"
- result=CELEvalError(err,KeyError,None,tree=tree)
- # TODO: Not sure this is needed...
+ result_value=CELEvalError(err,KeyError,None,tree=tree)elifisinstance(member,celpy.celtypes.MessageType):
+ # NOTE: Message's don't have a "default None" behavior: they raise an exception.self.logger.debug("member_dot(%r, %r)",member,property_name)
- result=member.get(property_name)
+ result_value=member.get(property_name)# TODO: Future Expansion, handle Protobuf message package...# elif isinstance(member, celpy.celtypes.PackageType):# if property_name in member:
- # result = member[property_name]
+ # result_value = member[property_name]# else:# err = f"no such message {property_name!r} in package {member}"
- # result = CELEvalError(err, KeyError, None, tree=tree)
+ # result_value = CELEvalError(err, KeyError, None, tree=tree)elifisinstance(member,celpy.celtypes.MapType):
- # Syntactic sugar: a.b is a["b"] when a is a mapping.
+ # Syntactic sugar: a.b is a["b"] when ``a`` is a mapping.try:
- result=member[property_name]
+ result_value=member[property_name]exceptKeyError:err=f"no such member in mapping: {property_name!r}"
- result=CELEvalError(err,KeyError,None,tree=tree)
+ result_value=CELEvalError(err,KeyError,None,tree=tree)else:err=f"{member!r} with type: '{type(member)}' does not support field selection"
- result=CELEvalError(err,TypeError,None,tree=tree)
- returnresult
- member "." IDENT "(" ")" -- used for a several timestamp operations. """sub_expr:CELFunction
- result:Result
+ result_value:Resultreduction:ResultCELBoolFunction=Callable[[celpy.celtypes.BoolType,Result],celpy.celtypes.BoolType
@@ -2313,9 +2649,13 @@
Source code for celpy.evaluation
"all","exists","exists_one",
+ # Extensions to CEL..."reduce","min",}:
+ # TODO: These can be refactored to share the macro_xyz() functions
+ # used by Transpiled code.
+
member_list=cast(celpy.celtypes.ListType,self.visit(member_tree))ifisinstance(member_list,CELEvalError):
@@ -2326,13 +2666,13 @@
count=sum(1forvalueinmember_listifbool(sub_expr(value)))returncelpy.celtypes.BoolType(count==1)
+ # Not formally part of CEL...elifmethod_name_token.value=="reduce":# Apply a function to reduce the list to a single value.# The `tree` is a `member_dot_arg` construct with (member, method_name, args)
@@ -2375,6 +2716,7 @@
Source code for celpy.evaluation
reduction=reduce(reduce_expr,member_list,initial_value)returnreduction
+ # Not formally part of CEL...elifmethod_name_token.value=="min":# Special case of "reduce()"# with <member>.min() -> <member>.reduce(r, i, int_max, r < i ? r : i)
@@ -2396,19 +2738,19 @@
Locating an item in a Mapping or List """
- func=self.functions["_[_]"]
+ # func = self.functions["_[_]"] # Refactor ``self.functions`` into an Activation
+ func=self.activation.resolve_function("_[_]")values=self.visit_children(tree)member,index=valuestry:
@@ -2450,7 +2793,7 @@
list_lit : "[" [exprlist] "]" map_lit : "{" [mapinits] "}"
- TODO: Refactor into separate methods to skip this complex elif chain.
- top-level :py:meth:`primary` is similar to :py:meth:`method`.
- Each of the individual rules then works with a tree instead of a child of the
- primary tree.
+ .. TODO:: Refactor into separate methods to skip these complex elif chain.
+
+ Top-level :py:meth:`primary` is similar to :py:meth:`method`.
+ Each of the individual rules then works with a tree instead of a child of the
+ primary tree.
- This includes function-like macros: has() and dyn().
+ This includes function-like macros: ``has()`` and ``dyn()``. These are special cases and cannot be overridden. """
- result:Result
+ result_value:Resultname_token:lark.Tokeniflen(tree.children)!=1:raiseCELSyntaxError(
@@ -2555,18 +2899,18 @@
Source code for celpy.evaluation
iflen(child.children)==0:# Empty list# TODO: Refactor into type_eval()
- result=celpy.celtypes.ListType()
+ result_value=celpy.celtypes.ListType()else:# exprlist to be packaged as List.values=self.visit_children(child)
- result=values[0]
- returnresult
+ result_value=values[0]
+ returnresult_valueelifchild.data=="map_lit":iflen(child.children)==0:# Empty mapping# TODO: Refactor into type_eval()
- result=celpy.celtypes.MapType()
+ result_value=celpy.celtypes.MapType()else:# mapinits (a sequence of key-value tuples) to be packaged as a dict.# OR. An CELEvalError in case of ValueError caused by duplicate keys.
@@ -2574,30 +2918,36 @@
Source code for celpy.evaluation
# TODO: Refactor into type_eval()try:values=self.visit_children(child)
- result=values[0]
+ result_value=values[0]exceptValueErrorasex:
- result=CELEvalError(ex.args[0],ex.__class__,ex.args,tree=tree)
+ result_value=CELEvalError(
+ ex.args[0],ex.__class__,ex.args,tree=tree
+ )exceptTypeErrorasex:
- result=CELEvalError(ex.args[0],ex.__class__,ex.args,tree=tree)
- returnresult
+ result_value=CELEvalError(
+ ex.args[0],ex.__class__,ex.args,tree=tree
+ )
+ returnresult_valueelifchild.datain("dot_ident","dot_ident_arg"):# "." IDENT ["(" [exprlist] ")"]# Leading "." means the name is resolved in the root scope **only**.
- # No searching through alterantive packages.
+ # No searching through alternative packages.# len(child) == 1 -- "." IDENT# len(child) == 2 -- "." IDENT "(" exprlist ")" -- TODO: Implement dot_ident_arg.values=self.visit_children(child)name_token=cast(lark.Token,values[0])# Should not be a Function, should only be a Result
- # TODO: implement dot_ident_arg uses function_eval().
+ # TODO: implement dot_ident_arg using ``function_eval()``, which should match this code.try:
- result=cast(
+ result_value=cast(Result,self.ident_value(name_token.value,root_scope=True))exceptKeyErrorasex:
- result=CELEvalError(ex.args[0],ex.__class__,ex.args,tree=tree)
- returnresult
+ result_value=CELEvalError(
+ ex.args[0],ex.__class__,ex.args,tree=tree
+ )
+ returnresult_valueelifchild.data=="ident_arg":# IDENT ["(" [exprlist] ")"]
@@ -2639,14 +2989,14 @@
Source code for celpy.evaluation
# Should not be a Function.# Generally Result object (i.e., a variable)# Could be an Annotation object (i.e., a type) for protobuf messages
- result=cast(Result,self.ident_value(name_token.value))
+ result_value=cast(Result,self.ident_value(name_token.value))exceptKeyErrorasex:err=(f"undeclared reference to '{name_token}' "f"(in activation '{self.activation}')")
- result=CELEvalError(err,ex.__class__,ex.args,tree=tree)
- returnresult
+ result_value=CELEvalError(err,ex.__class__,ex.args,tree=tree)
+ returnresult_valueelse:raiseCELSyntaxError(
@@ -2657,7 +3007,7 @@
exceptStopIteration:pass# There are no CELEvalError values in the result, so we can narrow the domain.
- result=celpy.celtypes.ListType(cast(List[celpy.celtypes.Value],values))
- returnresult
Extract the key expr's and value expr's to a list of pairs. This raises an exception on a duplicate key.
- TODO: Is ``{'a': 1, 'b': 2/0}['a']`` a meaningful result in CEL?
- Or is this an error because the entire member is erroneous?
+ .. TODO:: CEL question.
+ Is ``{'a': 1, 'b': 2/0}['a']`` a meaningful result in CEL?
+ Or is this an error because the entire member object is erroneous?
+
+ .. TODO:: Refactor to use MapType([(key, value),...]) init, which checks for duplicates.
+
+ This simplifies to ``celpy.celtypes.MapType(pairs)`` """
- result=celpy.celtypes.MapType()
+ result_value=celpy.celtypes.MapType()# Not sure if this cast is sensible. Should a CELEvalError propagate up from the# sub-expressions? See the error check in :py:func:`exprlist`.keys_values=cast(List[celpy.celtypes.Value],self.visit_children(tree))pairs=zip(keys_values[0::2],keys_values[1::2])forkey,valueinpairs:
- ifkeyinresult:
+ ifkeyinresult_value:raiseValueError(f"Duplicate key {key!r}")
- result[key]=value
+ result_value[key]=value
+
+ returnresult_value
+
+
+
+
+# GLOBAL activation used by Transpiled code.
+# This slightly simplifies the exception handling, by using a 1-argument function
+# to compute a Value or a CELEvalError.
+
+the_activation:Activation
+
+
+
+[docs]
+defresult(activation:Activation,cel_expr:Callable[[Activation],Result])->Result:
+"""
+ Implements "checked exception" handling for CEL expressions transpiled to Python.
+ An expression must be wrapped by a lambda.
+ The lambda is evaluated by this function; a subset of Python exceptions become ``CELEvalError`` objects.
+
+ >>> some_activation = Activation()
+
+ Within the CEL transpiled code, we can now use code like this...
+
+ >>> expr = lambda activation: 355 / 0
+ >>> result(some_activation, expr)
+ CELEvalError(*('divide by zero', <class 'ZeroDivisionError'>, ('division by zero',)))
+
+ The exception becomes an object.
+ """
+
+ value:Result
+ try:
+ value=cel_expr(activation)
+ except(
+ ValueError,
+ KeyError,
+ TypeError,
+ ZeroDivisionError,
+ OverflowError,
+ IndexError,
+ NameError,
+ )asex:
+ ex_message={
+ ValueError:"return error for overflow",
+ KeyError:f"no such member in mapping: {ex.args[0]!r}",
+ TypeError:"no such overload",
+ ZeroDivisionError:"divide by zero",
+ OverflowError:"return error for overflow",
+ IndexError:"invalid_argument",
+ UnicodeDecodeError:"invalid UTF-8",
+ NameError:f"undeclared reference to {ex.args[0]!r} (in container {ex.args[1:]!r})",
+ }[ex.__class__]
+ _,_,tb=sys.exc_info()
+ value=CELEvalError(ex_message,ex.__class__,ex.args).with_traceback(tb)
+ value.__cause__=ex
+ logger.debug("result = %r",value)
+ returnvalue
+
+
+
+
+[docs]
+defmacro_map(
+ activation:Activation,
+ bind_variable:str,
+ cel_expr:Callable[[Activation],celpy.celtypes.Value],
+ cel_gen:Callable[[Activation],Iterable[Activation]],
+)->Result:
+"""The results of a source.map(v, expr) macro: a list of values."""
+ activations=(
+ activation.nested_activation(vars={bind_variable:cast(Result,_value)})
+ for_valueincel_gen(activation)
+ )
+ returncelpy.celtypes.ListType(map(cel_expr,activations))
+
+
+
+
+[docs]
+defmacro_filter(
+ activation:Activation,
+ bind_variable:str,
+ cel_expr:Callable[[Activation],celpy.celtypes.Value],
+ cel_gen:Callable[[Activation],Iterable[Activation]],
+)->Result:
+"""The results of a source.filter(v, expr) macro: a list of values."""
+ r:list[celpy.celtypes.Value]=[]
+ forvalueincel_gen(activation):
+ f=cel_expr(
+ activation.nested_activation(vars={bind_variable:cast(Result,value)})
+ )
+ ifbool(f):
+ r.append(cast(celpy.celtypes.Value,value))
+ returncelpy.celtypes.ListType(iter(r))
+
+
+
+
+[docs]
+defmacro_exists_one(
+ activation:Activation,
+ bind_variable:str,
+ cel_expr:Callable[[Activation],celpy.celtypes.Value],
+ cel_gen:Callable[[Activation],Iterable[Activation]],
+)->Result:
+"""The results of a source.exists_one(v, expr) macro: a list of values.
+
+ Note the short-circuit concept.
+ Count the True; Break on an Exception
+ """
+ count=0
+ activations=(
+ activation.nested_activation(vars={bind_variable:cast(Result,_value)})
+ for_valueincel_gen(activation)
+ )
+ forresultinfilter(cel_expr,activations):
+ count+=1ifbool(result)else0
+ returncelpy.celtypes.BoolType(count==1)
+
+
+
+
+[docs]
+defmacro_exists(
+ activation:Activation,
+ bind_variable:str,
+ cel_expr:Callable[[Activation],celpy.celtypes.Value],
+ cel_gen:Callable[[Activation],Iterable[Activation]],
+)->Result:
+"""The results of a source.exists(v, expr) macro: a list of values."""
+ activations=(
+ activation.nested_activation(vars={bind_variable:cast(Result,_value)})
+ for_valueincel_gen(activation)
+ )
+ returncelpy.celtypes.BoolType(
+ reduce(
+ cast(
+ Callable[[celpy.celtypes.BoolType,Result],celpy.celtypes.BoolType],
+ celpy.celtypes.logical_or,
+ ),
+ (result(act,cel_expr)foractinactivations),
+ celpy.celtypes.BoolType(False),
+ )
+ )
+
+
+
+
+[docs]
+defmacro_all(
+ activation:Activation,
+ bind_variable:str,
+ cel_expr:Callable[[Activation],celpy.celtypes.Value],
+ cel_gen:Callable[[Activation],Iterable[Activation]],
+)->Result:
+"""The results of a source.all(v, expr) macro: a list of values."""
+ activations=(
+ activation.nested_activation(vars={bind_variable:cast(Result,_value)})
+ for_valueincel_gen(activation)
+ )
+ returncelpy.celtypes.BoolType(
+ reduce(
+ cast(
+ Callable[[celpy.celtypes.BoolType,Result],celpy.celtypes.BoolType],
+ celpy.celtypes.logical_and,
+ ),
+ (result(act,cel_expr)foractinactivations),
+ celpy.celtypes.BoolType(True),
+ )
+ )
+[docs]
+classTranspiler:
+"""
+ Transpile the CEL construct(s) to Python functions.
+ This is a **Facade** that wraps two visitor subclasses to do two phases
+ of transpilation.
+
+ The resulting Python code can be used with ``compile()`` and ``exec()``.
+
+ :Phase I:
+ The easy transpilation.
+ It builds simple text expressions for each node of the AST.
+ This sets aside exception-checking code including short-circuit logic operators and macros.
+ This decorates the AST with transpiled Python where possible.
+ It can also decorate with ``Template`` objects that require text from children.
+
+ :Phase II:
+ Collects a sequence of statements.
+ All of the exception-checking for short-circuit logic operators and macros is packaged as lambdas
+ that may (or may not) be evaluated.
+
+ Ideally, there could be a ``Transpiler`` ABC, and the ``PythonTranspiler`` defined as a subclass.
+ Pragmatically, we can't see any other sensible transpilation.
+
+ .. rubric:: Exception Checking
+
+ To handle ``2 / 0 || true``, the ``||``, ``&&``, and ``?:`` operators
+ the generated code creates lambdas to avoid execution where possible.
+ An alternative is a Monad-like structure to bottle up an exception, silencing it if it's unused.
+
+ .. rubric:: Identifiers
+
+ Identifiers have three meanings:
+
+ - An object. This is either a variable provided in an ``Activation`` or a function provided
+ when building an execution. Objects also have type annotations.
+
+ - A type annotation without an object. This can used to build protobuf messages.
+
+ - A macro name. The ``member_dot_arg`` construct (e.g., ``member.map(v, expr)``) may have a macro instead of a method.
+ Plus the ``ident_arg`` construct may be a ``dyn()`` or ``has()`` macro instead of a function
+
+ Other than macros, a name maps to an ``Referent`` instance.
+ This will have an annotation and -- perhaps -- an associated object.
+
+ .. important MACROS ARE SPECIAL
+
+ They aren't simple functions.
+
+ The macros do not simply visit their children to perform evaluation.
+
+ There's a bind variable and a function with the bind variable.
+ This isn't **trivially** moved from expression stack to statements.
+
+ There are two functions that are macro-like:
+
+ - ``dyn()`` does effectively nothing.
+ It visits its children, but also provides progressive type resolution
+ through detailed type annotation of the AST.
+
+ - ``has()`` attempts to visit the child and does a boolean transformation
+ on the resulting exception or value.
+ This is a macro because it doesn't raise the exception for a missing
+ member item reference, but instead maps any exception to ``False``.
+ It doesn't return the value found for a member item reference; instead, it maps
+ successfully finding a member to ``True``.
+
+ The member and expression list of a macro are transformed into lambdas for use by
+ special ``macro_{name}`` functions. These functions provided the necessary generator
+ expression to provide CEL semantics.
+
+ Names have nested paths. For example, ``a.b.c`` is a mapping ``a``, that contains a mapping, ``b``,
+ that contains a name ``c``.
+
+ The :py:meth:`member` method implements the macro evaluation behavior.
+ It does not **always** trivially descend into the children.
+ In the case of macros, the member evaluates one child tree in the presence
+ of values from another child tree using specific variable binding in a kind
+ of stack frame.
+ """
+
+ logger=logging.getLogger("celpy.Transpiler")
+
+
+[docs]
+ def__init__(
+ self,
+ ast:TranspilerTree,
+ activation:Activation,
+ )->None:
+"""
+ Create the Transpiler for an AST with specific variables and functions.
+
+ :param ast: The AST to transpile.
+ :param activation: An activation with functions and types to use.
+ """
+ self.ast=ast
+ self.base_activation=activation
+ self.activation=self.base_activation
+
+ self.logger.debug("Transpiler activation: %r",self.activation)
- returnresult
+ # self.logger.debug("functions: %r", self.functions) # Refactor ``self.functions`` into an Activation
+
+
+[docs]
+ deftranspile(self)->None:
+"""Two-phase transpilation.
+
+ 1. Decorate AST with the most constructs.
+ 2. Expand into statements for lambdas that wrap checked exceptions.
+ """
+ phase_1=Phase1Transpiler(self)
+ phase_1.visit(self.ast)
+
+ phase_2=Phase2Transpiler(self)
+ phase_2.visit(self.ast)
+
+ statements=phase_2.statements(self.ast)
+
+ # The complete sequence of statements and the code object.
+ self.source_text="\n".join(statements)
+ self.executable_code=compile(self.source_text,"<string>","exec")
+
+
+
+[docs]
+ defevaluate(self,context:Context)->celpy.celtypes.Value:
+ ifcontext:
+ self.activation=self.base_activation.clone()
+ self.activation.identifiers.load_values(context)
+ else:
+ self.activation=self.base_activation
+ self.logger.debug("Activation: %r",self.activation)
+
+ # Global for the top-level ``CEL = result(base_activation, ...)`` statement.
+ evaluation_globals=(
+ celpy.evaluation.result.__globals__
+ )# the ``evaluation`` moodule
+ evaluation_globals["base_activation"]=self.activation
+ try:
+ exec(self.executable_code,evaluation_globals)
+ value=cast(celpy.celtypes.Value,evaluation_globals["CEL"])
+ ifisinstance(value,CELEvalError):
+ raisevalue
+ returnvalue
+ exceptExceptionasex:
+ # A Python problem during ``exec()``
+ self.logger.error("Internal error: %r",ex)
+ raiseCELEvalError("evaluation error",type(ex),ex.args)
+
+
+
+
+
+[docs]
+classPhase1Transpiler(lark.visitors.Visitor_Recursive):
+"""
+ Decorate all nodes with transpiled Python code, where possible.
+ For short-circuit operators or macros, where a "checked exception" is required,
+ a simple ``ex_{n}`` name is present, and separate statements are provided as
+ a decoration to handle the more complicated cases.
+
+ Each construct has an associated ``Template``.
+ For the simple cases, the transpiled value is the entire expression.
+
+ >>> from unittest.mock import Mock
+ >>> source = "7 * (3 + 3)"
+ >>> parser = celpy.CELParser()
+ >>> tree = parser.parse(source)
+ >>> tp = Phase1Transpiler(Mock(base_activation=celpy.Activation()))
+ >>> _ = tp.visit(tree)
+ >>> tree.transpiled
+ 'operator.mul(celpy.celtypes.IntType(7), operator.add(celpy.celtypes.IntType(3), celpy.celtypes.IntType(3)))'
+
+ Some constructs wrap macros or short-circuit logic, and require a more sophisticated execution.
+ There will be "checked exceptions", returned as values.
+ This requires statements with lambdas that can be wrapped by the ``result()`` function.
+
+ The ``Phase2Transpiler`` does this transformation from expressions to a sequence of statements.
+ """
+
+
+[docs]
+ defvisit(self,tree:TranspilerTree)->TranspilerTree:# type: ignore[override]
+"""Initialize the decorations for each node."""
+ tree.expr_number=self.expr_number
+ # tree.transpiled = f"ex_{tree.expr_number}(activation)" # Default, will be replaced.
+ # tree.checked_exception: Union[str, None] = None # Optional
+ self.expr_number+=1
+ returnsuper().visit(tree)# type: ignore[return-value]
+
+
+
+[docs]
+ deffunc_name(self,label:str)->str:
+"""
+ Provide a transpiler-friendly name for the function.
+
+ Some internally-defined functions appear to come from ``_operator`` module.
+ We need to rename some ``celpy`` functions to be from ``operator``.
+
+ Some functions -- specifically lt, le, gt, ge, eq, ne -- are wrapped ``boolean(operator.f)``
+ obscuring their name.
+ """
+ try:
+ # func = self.functions[label] # Refactor ``self.functions`` into an Activation
+ func=self.activation.resolve_function(label)
+ exceptKeyError:
+ returnf"CELEvalError('unbound function', KeyError, ({label!r},))"
+ module={"_operator":"operator"}.get(func.__module__,func.__module__)
+ returnf"{module}.{func.__qualname__}"
+[docs]
+ defmember_dot(self,tree:TranspilerTree)->None:
+"""
+ member_dot : member "." IDENT
+
+ .. important::
+
+ The ``member`` can be any of a variety of objects:
+
+ - ``NameContainer(Dict[str, Referent])``
+
+ - ``Activation``
+
+ - ``MapType(Dict[Value, Value])``
+
+ - ``MessageType(MapType)``
+
+ All of which define a ``get()`` method.
+ The nuance is the ``NameContainer`` is also a Python ``dict`` and there's an
+ overload issue between that ``get()`` and other ``get()`` definitions.
+
+ .. todo:: Define a new get_name(member, 'name') function do this, avoiding the ``get()`` method.
+ """
+ member_tree,property_name_token=cast(
+ Tuple[TranspilerTree,lark.Token],tree.children
+ )
+ template=Template("${left}.get('${right}')")
+ tree.transpiled=template.substitute(
+ left=member_tree.transpiled,right=property_name_token.value
+ )
+[docs]
+ defstatements(self,tree:TranspilerTree)->list[str]:
+"""
+ Appends the final CEL = ... statement to the sequence of statements,
+ and returns the transpiled code.
+
+ Two patterns:
+
+ 1. Top-most expr was a deferred template, and already is a lambda.
+ It will be a string of the form ``"ex_\\d+(activation)"``.
+
+ 2. Top-most expr was **not** a deferred template, and needs a lambda wrapper.
+ It will **not** be a simple ``"ex_\\d+"`` reference.
+ """
+ expr_pattern=re.compile(r"^(ex_\d+)\(\w+\)$")
+ ifmatch:=expr_pattern.match(tree.transpiled):
+ template=Template(
+ dedent("""\
+ CEL = celpy.evaluation.result(base_activation, ${lambda_name})
+ """)
+ )
+ final=template.substitute(n=tree.expr_number,lambda_name=match.group(1))
+ else:
+ template=Template(
+ dedent("""\
+ CEL = celpy.evaluation.result(base_activation, lambda activation: ${expr})
+ """)
+ )
+ final=template.substitute(n=tree.expr_number,expr=tree.transpiled)
+ returnself._statements+[final]
@@ -2801,7 +4260,7 @@
Source code for celpy.evaluation
-[docs]
+[docs]defcelstr(token:lark.Token)->celpy.celtypes.StringType:""" Evaluate a CEL string literal, expanding escapes to create a Python string.
@@ -2852,7 +4311,7 @@
Source code for celpy.evaluation
-[docs]
+[docs]defcelbytes(token:lark.Token)->celpy.celtypes.BytesType:""" Evaluate a CEL bytes literal, expanding escapes to create a Python bytes object.
@@ -2925,14 +4384,15 @@
+importsys
+fromcopyimportdeepcopy
+
+fromtypingimportList,Callable,Iterator,Union,Optional,Generic,TypeVar,TYPE_CHECKING
+
+ifTYPE_CHECKING:
+ from.lexerimportTerminalDef,Token
+ try:
+ importrich
+ exceptImportError:
+ pass
+ fromtypingimportLiteral
+
+###{standalone
+
+classMeta:
+
+ empty:bool
+ line:int
+ column:int
+ start_pos:int
+ end_line:int
+ end_column:int
+ end_pos:int
+ orig_expansion:'List[TerminalDef]'
+ match_tree:bool
+
+ def__init__(self):
+ self.empty=True
+
+
+_Leaf_T=TypeVar("_Leaf_T")
+Branch=Union[_Leaf_T,'Tree[_Leaf_T]']
+
+
+classTree(Generic[_Leaf_T]):
+"""The main tree class.
+
+ Creates a new tree, and stores "data" and "children" in attributes of the same name.
+ Trees can be hashed and compared.
+
+ Parameters:
+ data: The name of the rule or alias
+ children: List of matched sub-rules and terminals
+ meta: Line & Column numbers (if ``propagate_positions`` is enabled).
+ meta attributes: (line, column, end_line, end_column, start_pos, end_pos,
+ container_line, container_column, container_end_line, container_end_column)
+ container_* attributes consider all symbols, including those that have been inlined in the tree.
+ For example, in the rule 'a: _A B _C', the regular attributes will mark the start and end of B,
+ but the container_* attributes will also include _A and _C in the range. However, rules that
+ contain 'a' will consider it in full, including _A and _C for all attributes.
+ """
+
+ data:str
+ children:'List[Branch[_Leaf_T]]'
+
+ def__init__(self,data:str,children:'List[Branch[_Leaf_T]]',meta:Optional[Meta]=None)->None:
+ self.data=data
+ self.children=children
+ self._meta=meta
+
+ @property
+ defmeta(self)->Meta:
+ ifself._metaisNone:
+ self._meta=Meta()
+ returnself._meta
+
+ def__repr__(self):
+ return'Tree(%r, %r)'%(self.data,self.children)
+
+ def_pretty_label(self):
+ returnself.data
+
+ def_pretty(self,level,indent_str):
+ yieldf'{indent_str*level}{self._pretty_label()}'
+ iflen(self.children)==1andnotisinstance(self.children[0],Tree):
+ yieldf'\t{self.children[0]}\n'
+ else:
+ yield'\n'
+ forninself.children:
+ ifisinstance(n,Tree):
+ yield fromn._pretty(level+1,indent_str)
+ else:
+ yieldf'{indent_str*(level+1)}{n}\n'
+
+ defpretty(self,indent_str:str=' ')->str:
+"""Returns an indented string representation of the tree.
+
+ Great for debugging.
+ """
+ return''.join(self._pretty(0,indent_str))
+
+ def__rich__(self,parent:Optional['rich.tree.Tree']=None)->'rich.tree.Tree':
+"""Returns a tree widget for the 'rich' library.
+
+ Example:
+ ::
+ from rich import print
+ from lark import Tree
+
+ tree = Tree('root', ['node1', 'node2'])
+ print(tree)
+ """
+ returnself._rich(parent)
+
+ def_rich(self,parent):
+ ifparent:
+ tree=parent.add(f'[bold]{self.data}[/bold]')
+ else:
+ importrich.tree
+ tree=rich.tree.Tree(self.data)
+
+ forcinself.children:
+ ifisinstance(c,Tree):
+ c._rich(tree)
+ else:
+ tree.add(f'[green]{c}[/green]')
+
+ returntree
+
+ def__eq__(self,other):
+ try:
+ returnself.data==other.dataandself.children==other.children
+ exceptAttributeError:
+ returnFalse
+
+ def__ne__(self,other):
+ returnnot(self==other)
+
+ def__hash__(self)->int:
+ returnhash((self.data,tuple(self.children)))
+
+ defiter_subtrees(self)->'Iterator[Tree[_Leaf_T]]':
+"""Depth-first iteration.
+
+ Iterates over all the subtrees, never returning to the same node twice (Lark's parse-tree is actually a DAG).
+ """
+ queue=[self]
+ subtrees=dict()
+ forsubtreeinqueue:
+ subtrees[id(subtree)]=subtree
+ queue+=[cforcinreversed(subtree.children)
+ ifisinstance(c,Tree)andid(c)notinsubtrees]
+
+ delqueue
+ returnreversed(list(subtrees.values()))
+
+ defiter_subtrees_topdown(self):
+"""Breadth-first iteration.
+
+ Iterates over all the subtrees, return nodes in order like pretty() does.
+ """
+ stack=[self]
+ stack_append=stack.append
+ stack_pop=stack.pop
+ whilestack:
+ node=stack_pop()
+ ifnotisinstance(node,Tree):
+ continue
+ yieldnode
+ forchildinreversed(node.children):
+ stack_append(child)
+
+ deffind_pred(self,pred:'Callable[[Tree[_Leaf_T]], bool]')->'Iterator[Tree[_Leaf_T]]':
+"""Returns all nodes of the tree that evaluate pred(node) as true."""
+ returnfilter(pred,self.iter_subtrees())
+
+ deffind_data(self,data:str)->'Iterator[Tree[_Leaf_T]]':
+"""Returns all nodes of the tree whose data equals the given data."""
+ returnself.find_pred(lambdat:t.data==data)
+
+###}
+
+ defexpand_kids_by_data(self,*data_values):
+"""Expand (inline) children with any of the given data values. Returns True if anything changed"""
+ changed=False
+ foriinrange(len(self.children)-1,-1,-1):
+ child=self.children[i]
+ ifisinstance(child,Tree)andchild.dataindata_values:
+ self.children[i:i+1]=child.children
+ changed=True
+ returnchanged
+
+
+ defscan_values(self,pred:'Callable[[Branch[_Leaf_T]], bool]')->Iterator[_Leaf_T]:
+"""Return all values in the tree that evaluate pred(value) as true.
+
+ This can be used to find all the tokens in the tree.
+
+ Example:
+ >>> all_tokens = tree.scan_values(lambda v: isinstance(v, Token))
+ """
+ forcinself.children:
+ ifisinstance(c,Tree):
+ fortinc.scan_values(pred):
+ yieldt
+ else:
+ ifpred(c):
+ yieldc
+
+ def__deepcopy__(self,memo):
+ returntype(self)(self.data,deepcopy(self.children,memo),meta=self._meta)
+
+ defcopy(self)->'Tree[_Leaf_T]':
+ returntype(self)(self.data,self.children)
+
+ defset(self,data:str,children:'List[Branch[_Leaf_T]]')->None:
+ self.data=data
+ self.children=children
+
+
+ParseTree=Tree['Token']
+
+
+classSlottedTree(Tree):
+ __slots__='data','children','rule','_meta'
+
+
+defpydot__tree_to_png(tree:Tree,filename:str,rankdir:'Literal["TB", "LR", "BT", "RL"]'="LR",**kwargs)->None:
+ graph=pydot__tree_to_graph(tree,rankdir,**kwargs)
+ graph.write_png(filename)
+
+
+defpydot__tree_to_dot(tree:Tree,filename,rankdir="LR",**kwargs):
+ graph=pydot__tree_to_graph(tree,rankdir,**kwargs)
+ graph.write(filename)
+
+
+defpydot__tree_to_graph(tree:Tree,rankdir="LR",**kwargs):
+"""Creates a colorful image that represents the tree (data+children, without meta)
+
+ Possible values for `rankdir` are "TB", "LR", "BT", "RL", corresponding to
+ directed graphs drawn from top to bottom, from left to right, from bottom to
+ top, and from right to left, respectively.
+
+ `kwargs` can be any graph attribute (e. g. `dpi=200`). For a list of
+ possible attributes, see https://www.graphviz.org/doc/info/attrs.html.
+ """
+
+ importpydot# type: ignore[import-not-found]
+ graph=pydot.Dot(graph_type='digraph',rankdir=rankdir,**kwargs)
+
+ i=[0]
+
+ defnew_leaf(leaf):
+ node=pydot.Node(i[0],label=repr(leaf))
+ i[0]+=1
+ graph.add_node(node)
+ returnnode
+
+ def_to_pydot(subtree):
+ color=hash(subtree.data)&0xffffff
+ color|=0x808080
+
+ subnodes=[_to_pydot(child)ifisinstance(child,Tree)elsenew_leaf(child)
+ forchildinsubtree.children]
+ node=pydot.Node(i[0],style="filled",fillcolor="#%x"%color,label=subtree.data)
+ i[0]+=1
+ graph.add_node(node)
+
+ forsubnodeinsubnodes:
+ graph.add_edge(pydot.Edge(node,subnode))
+
+ returnnode
+
+ _to_pydot(tree)
+ returngraph
+
+
+
+
+
+
+
+
\ No newline at end of file
diff --git a/docs/build/html/_plantuml/01/01f21512c6b94e19efccb4676b518bed63780ddd.png b/docs/build/html/_plantuml/01/01f21512c6b94e19efccb4676b518bed63780ddd.png
new file mode 100644
index 0000000..080aa3a
Binary files /dev/null and b/docs/build/html/_plantuml/01/01f21512c6b94e19efccb4676b518bed63780ddd.png differ
diff --git a/docs/build/html/_plantuml/16/165a23eb11f806bdd308ffac47a78125152ade58.png b/docs/build/html/_plantuml/16/165a23eb11f806bdd308ffac47a78125152ade58.png
new file mode 100644
index 0000000..b02b306
Binary files /dev/null and b/docs/build/html/_plantuml/16/165a23eb11f806bdd308ffac47a78125152ade58.png differ
diff --git a/docs/build/html/_plantuml/59/59894b154086c2a2ebe8ae46d280279ecce7cf8f.png b/docs/build/html/_plantuml/59/59894b154086c2a2ebe8ae46d280279ecce7cf8f.png
new file mode 100644
index 0000000..16f9a9c
Binary files /dev/null and b/docs/build/html/_plantuml/59/59894b154086c2a2ebe8ae46d280279ecce7cf8f.png differ
diff --git a/docs/build/html/_plantuml/60/60c95188891b5d1267db67bb61a17e9fd7c08060.png b/docs/build/html/_plantuml/60/60c95188891b5d1267db67bb61a17e9fd7c08060.png
new file mode 100644
index 0000000..cb501f6
Binary files /dev/null and b/docs/build/html/_plantuml/60/60c95188891b5d1267db67bb61a17e9fd7c08060.png differ
diff --git a/docs/build/html/_plantuml/60/60c95188891b5d1267db67bb61a17e9fd7c08060.png.newa1v5axm0 b/docs/build/html/_plantuml/60/60c95188891b5d1267db67bb61a17e9fd7c08060.png.newa1v5axm0
new file mode 100644
index 0000000..e69de29
diff --git a/docs/build/html/_plantuml/60/60c95188891b5d1267db67bb61a17e9fd7c08060.png.newlpv3pfrv b/docs/build/html/_plantuml/60/60c95188891b5d1267db67bb61a17e9fd7c08060.png.newlpv3pfrv
new file mode 100644
index 0000000..e69de29
diff --git a/docs/build/html/_plantuml/60/60c95188891b5d1267db67bb61a17e9fd7c08060.png.newv2qg6lg_ b/docs/build/html/_plantuml/60/60c95188891b5d1267db67bb61a17e9fd7c08060.png.newv2qg6lg_
new file mode 100644
index 0000000..e69de29
diff --git a/docs/build/html/_plantuml/a8/a8a77cea7a1f2555dbb1cb6cea10dc21777c76bf.png b/docs/build/html/_plantuml/a8/a8a77cea7a1f2555dbb1cb6cea10dc21777c76bf.png
new file mode 100644
index 0000000..96c3652
Binary files /dev/null and b/docs/build/html/_plantuml/a8/a8a77cea7a1f2555dbb1cb6cea10dc21777c76bf.png differ
diff --git a/docs/build/html/_plantuml/a8/a8a77cea7a1f2555dbb1cb6cea10dc21777c76bf.png.new1cmjxeth b/docs/build/html/_plantuml/a8/a8a77cea7a1f2555dbb1cb6cea10dc21777c76bf.png.new1cmjxeth
new file mode 100644
index 0000000..e69de29
diff --git a/docs/build/html/_plantuml/a8/a8a77cea7a1f2555dbb1cb6cea10dc21777c76bf.png.new4bry_4gw b/docs/build/html/_plantuml/a8/a8a77cea7a1f2555dbb1cb6cea10dc21777c76bf.png.new4bry_4gw
new file mode 100644
index 0000000..e69de29
diff --git a/docs/build/html/_plantuml/a8/a8a77cea7a1f2555dbb1cb6cea10dc21777c76bf.png.new58drlmt5 b/docs/build/html/_plantuml/a8/a8a77cea7a1f2555dbb1cb6cea10dc21777c76bf.png.new58drlmt5
new file mode 100644
index 0000000..e69de29
diff --git a/docs/build/html/_plantuml/ac/ac649ac296fc6bea0cee4f6cb2d92533640e6763.png b/docs/build/html/_plantuml/ac/ac649ac296fc6bea0cee4f6cb2d92533640e6763.png
new file mode 100644
index 0000000..391d74a
Binary files /dev/null and b/docs/build/html/_plantuml/ac/ac649ac296fc6bea0cee4f6cb2d92533640e6763.png differ
diff --git a/docs/build/html/_plantuml/c9/c970d97dc7e0a41eb7666fff5d466440fc8d67cf.png b/docs/build/html/_plantuml/c9/c970d97dc7e0a41eb7666fff5d466440fc8d67cf.png
new file mode 100644
index 0000000..75c2ffd
Binary files /dev/null and b/docs/build/html/_plantuml/c9/c970d97dc7e0a41eb7666fff5d466440fc8d67cf.png differ
diff --git a/docs/build/html/_sources/api.rst.txt b/docs/build/html/_sources/api.rst.txt
index da47f98..7515b4f 100644
--- a/docs/build/html/_sources/api.rst.txt
+++ b/docs/build/html/_sources/api.rst.txt
@@ -5,11 +5,30 @@
.. _`api`:
###########
-CEL-Py API
+API
###########
Details of the CEL-Python implementation and the API to the various components.
+The following modules are described here:
+
+- celpy_ (The ``__init__.py`` file defines this.)
+
+- \_\_main\_\_ (The CLI main application.)
+
+- adapter_
+
+- c7nlib_
+
+- celparser_
+
+- celtypes_
+
+- evaluation_
+
+``celpy``
+=========
+
.. automodule:: celpy.__init__
``__main__``
@@ -17,18 +36,28 @@ Details of the CEL-Python implementation and the API to the various components.
.. automodule:: celpy.__main__
-celtypes
-=========
+``adapter``
+===========
-.. automodule:: celpy.celtypes
+.. automodule:: celpy.adapter
-evaluation
-==========
+``c7nlib``
+===========
-.. automodule:: celpy.evaluation
+.. automodule:: celpy.c7nlib
-parser
-======
+``celparser``
+=============
.. automodule:: celpy.celparser
+``celtypes``
+=============
+
+.. automodule:: celpy.celtypes
+
+``evaluation``
+==============
+
+.. automodule:: celpy.evaluation
+
diff --git a/docs/build/html/_sources/c7n_functions.rst.txt b/docs/build/html/_sources/c7n_functions.rst.txt
index 62b8c6d..edc12bd 100644
--- a/docs/build/html/_sources/c7n_functions.rst.txt
+++ b/docs/build/html/_sources/c7n_functions.rst.txt
@@ -5,15 +5,13 @@ C7N Functions Required
######################
-This survey of C7N filter clauses is based on source code and
-on an analysis of working policies. The required functions
-are grouped into four clusters, depending on the presence of absence of
-the "op" operator clause, and the number of resource types using the feature.
+This survey of C7N filter clauses is based on source code and on an analysis of working policies.
+The required functions are grouped into four clusters, depending on the presence of absence of the "op" operator clause, and the number of resource types using the feature.
Within each group, they are ranked in order of popularity.
For each individual type of filter clause, we provide the following details:
-- The C7N sechema definition.
+- The C7N schema definition.
- The resource types where the filter type is used.
@@ -43,7 +41,7 @@ interface.
1. Separate from C7N. The CEL processing is outside C7N, and capable of standing alone.
CEL is focused on Protobuf (and JSON) objects.
The interface to C7N is via the :mod:`c7nlib` library of functions. These do **not** depend
- on imports from the C7N project, but rely on a `CELFilter` class offering specific methods.
+ on imports from the C7N project, but rely on a ``CELFilter`` class offering specific methods.
Access to C7N objects and their associated methods is limited to the features exposed
through the function library and the expected class definition.
diff --git a/docs/build/html/_sources/cli.rst.txt b/docs/build/html/_sources/cli.rst.txt
index 3cb4b02..96a5319 100644
--- a/docs/build/html/_sources/cli.rst.txt
+++ b/docs/build/html/_sources/cli.rst.txt
@@ -6,50 +6,302 @@
CLI Use of CEL-Python
######################
-We can read JSON directly from stdin, making this a bit like JQ.
+While CEL-Python's primary use case is integration into an DSL-based application to provide expressions with a uniform syntax and well-defined semantics.
+The expression processing capability is also available as a CLI implemented in the ``celpy`` package.
+
+SYNOPSIS
+========
::
- % PYTHONPATH=src python -m celpy '.this.from.json * 3 + 3' <, --arg
+
+ Define argument variables, types, and (optional) values.
+ If the argument value is omitted, then an environment variable will be examined to find the value.
+ For example, ``--arg HOME:string`` makes the :envvar:`HOME` environment variable's value available to the CEL expression.
+
+.. option:: -b, --boolean
+
+ Return a status code value based on the boolean output.
+
+ true has a status code of 0
+
+ false has a statis code of 1
+
+ Any exception has a stats code of 2
+
+.. option:: -n, --null-input
+
+ Do not read JSON input from stdin
+
+.. option:: -s, --slurp
+
+ Treat all input as a single JSON document.
+ The default is to treat each line of input as a separate NLJSON document.
+
+.. option:: -i, --interactive
+
+ Operate interactively from a ``CEL>`` prompt.
+ In :option:`-i` mode, the rest of the options are ignored.
+
+.. option:: -p, --json-package
+
+ Each NDJSON input (or the single input in :option:`-s` mode)
+ is a CEL package.
+
+.. option:: -d, --json-document
+
+ Each NDJSON input (or the single input in :option:`-s` mode)
+ is a separate CEL variable.
+
+.. option:: -f , --format
+
+ Use Python formating instead of JSON conversion of results;
+ Example ``--format .6f`` to format a ``DoubleType`` result
+
+.. option:: expr
+
+ A CEL expression to evaluate.
+
+DESCRIPTION
+============
+
+This provides shell-friendly expression processing.
+It follows patterns from several programs.
+
+:jq:
+ The ``celpy`` application will read newline-delimited JSON
+ from stdin.
+ It can also read a single, multiline JSON document in ``--slurp`` mode.
+
+ This will evaluate the expression for each JSON document.
+
+ .. note::
+
+ ``jq`` uses ``.`` to refer the current document. By setting a package
+ name of ``"jq"`` with the :option:`-p` option, e.g., ``-p jq``,
+ and placing the JSON object in the same package, we achieve
+ similar syntax.
+
+:expr:
+ The ``celpy`` application does everything ``expr`` does, but the syntax is different.
+
+ The output of comparisons in ``celpy`` is boolean, where by default.
+ The ``expr`` program returns an integer 1 or 0.
+ Use the :option:`-f` option, for example, ``-f 'd'`` to see decimal output instead of Boolean text values.
+
+:test:
+ This does what ``test`` does using CEL syntax.
+ The ``stat()`` function retrieves a mapping with various file status values.
+
+ Use the :option:`-b` option to set the exit status code from the Boolean result.
+
+ A ``true`` value becomes a 0 exit code.
+
+ A ``false`` value becomes a 1 exit code.
+
+:bc:
+ THe little-used linux ``bc`` application has several complex function definitions and other programming support.
+ CEL can evaluate some ``bc``\\ -like expressions.
+ It could be extended to mimic ``bc``.
+
+Additionally, in :option:`--interactive` mode,
+there's a REPL with a ``CEL>`` prompt.
+
+Arguments, Types, and Namespaces
+---------------------------------
+
+The :option:`--arg` options must provide a variable name and type.
+CEL objects rely on the :py:mod:`celpy.celtypes` definitions.
+
+Because of the close association between CEL and protobuf, some well-known protobuf types
+are also supported.
+
+The value for a variable is optional.
+If it is not provided, then the variable is presumed to be an environment variable.
+While many environment variables are strings, the type is still required.
+For example, use ``--arg HOME:string`` to get the value of the :envvar:`HOME` environment variable.
+
+FILES
+======
+
+By default, JSON documents are read from stdin in NDJSON format (http://jsonlines.org/, http://ndjson.org/).
+For each JSON document, the expression is evaluated with the document in a default
+package. This allows `.name` to pick items from the document.
+
+By default, the output is JSON serialized.
+This means strings will be JSON-ified and have quotes.
+Using the :option:`-f` option will expect a single, primitive type that can be formatting using Python's string formatting mini-language.
+
+ENVIRONMENT VARIABLES
+=====================
+
+Enhanced logging is available when :envvar:`CEL_TRACE` is defined.
+This is quite voluminous; tracing most pieces of the AST during evaluation.
+
+CONFIGURATION
+=============
+
+Logging configuration is read from the ``celpy.toml`` file.
+See :ref:`configuration` for details.
+
+EXIT STATUS
+===========
+
+Normally, it's zero.
+
+When the :option:`-b` option is used then the final expression determines the status code.
+
+A value of ``true`` returns 0.
+
+A value of ``false`` returns 1.
+
+Other values or an evaluation error exception will return 2.
+
+EXAMPLES
+========
+
+We can read JSON directly from stdin, making this a bit like the **jq** application.
+We provide a JQ expression, ``'.this.from.json * 3 + 3'``, and a JSON document.
+The standard output is the computed result.
+
+.. code-block:: bash
+
+ % python -m celpy '.this.from.json * 3 + 3' < {"this": {"from": {"json": 13}}}
heredoc> EOF
42
+The default behavior is to read and process stdin, where each line is a separate JSON document.
+This is the Newline-Delimited JSON format.
+(See https://jsonlines.org and https://github.com/ndjson/ndjson-spec).
+
+The ``-s/--slurp`` treats the stdin as a single JSON document, spread over multiple lines.
+This parallels the way the the **jq** application handles JSON input.
+
+We can avoid reading stdin by using the ``-n/--null-input`` option.
+This option will evaluate the expression using only command-line argument values.
+
It's also a desk calculator.
-::
+.. code-block:: bash
% python -m celpy -n '355.0 / 113.0'
3.1415929203539825
-And, yes, this has a tiny advantage over ``python -c '355/113'``. Most notably, the ability
-to embed Google CEL into other contexts where you don't *really* want Python's power.
-There's no CEL ``import`` or built-in ``exec()`` function to raise concerns.
+And, yes, this use case has a tiny advantage over ``python -c '355/113'``.
+Most notably, the ability to embed Google CEL into other contexts where you don't *really* want Python's power.
+There's no CEL ``import`` or built-in ``eval()`` function to raise security concerns.
-We can provide a ``-d`` option to define objects with particular data types, like JSON.
-This is particularly helpful for providing protobuf message definitions.
+We can provide a ``-a/--arg`` option to define a name in the current activation with particular data type.
+The expression, ``'x * 3 + 3'`` depends on a ``x`` variable, set by the ``-a`` option.
+Note the ``variable:type`` syntax for setting the type of the variable.
-::
+.. code-block:: bash
- % PYTHONPATH=src python -m celpy -n -ax:int=13 'x * 3 + 3'
+ % python -m celpy -n -ax:int=13 'x * 3 + 3'
42
-This command sets a variable ``x`` then evaluates the expression. And yes, this is what
-``expr`` does. CEL can do more. For example, floating-point math.
+This is what the bash ``expr`` command does.
+CEL can do more.
+For example, floating-point math.
+Here we've set two variables, ``x`` and ``tot``, before evaluating an expression.
-::
+.. code-block:: bash
- % PYTHONPATH=src python -m celpy -n -ax:double=113 -atot:double=355 '100. * x/tot'
+ % python -m celpy -n -ax:double=113 -atot:double=355 '100. * x/tot'
31.830985915492956
-We can also mimic the ``test`` command.
+If you omit the ``=`` from the ``-a`` option, then an environment variable's
+value will be bound to the variable name in the activation.
-::
+.. code-block:: bash
+
+ % TOTAL=41 python -m celpy -n -aTOTAL:int 'TOTAL + 1'
+ 42
+
+Since these operations involves explict type conversions, be aware of the possibility of syntax error exceptions.
+
+.. code-block:: bash
- % PYTHONPATH=src python -m celpy -n -ax:int=113 -atot:int=355 -b 'x > tot'
+ % TOTAL="not a number" python -m celpy -n -aTOTAL:int 'TOTAL + 1'
+ usage: celpy [-h] [-v] [-a ARG] [-n] [-s] [-i] [--json-package NAME] [--json-document NAME] [-b] [-f FORMAT] [expr]
+ celpy: error: argument -a/--arg: arg TOTAL:int value invalid for the supplied type
+
+
+
+We can also use this instead of the bash ``test`` command.
+We can bind values with the ``-a`` options and then compare them.
+The ``-b/--boolean`` option sets the status value based on the boolean result value.
+The output string is the CEL literal value ``false``.
+The status code is a "failure" code of 1.
+
+.. code-block:: bash
+
+ % python -m celpy -n -ax:int=113 -atot:int=355 -b 'x > tot'
false
% echo $?
1
-The intent is to provide a common implementation for aritmetic and logic.
+Here's another example that shows the ``stat()`` function to get filesystem status.
+
+.. code-block:: bash
+
+ % python -m celpy -n -aHOME 'HOME.stat()'
+ {"st_atime": "2025-07-06T20:27:21Z", "st_birthtime": "2006-11-27T18:30:03Z", "st_ctime": "2025-07-06T20:27:20Z", "st_dev": 16777234, "st_ino": 341035, "st_mtime": "2025-07-06T20:27:20Z", "st_nlink": 135, "st_size": 4320, "group_access": true, "user_access": true, "kind": "d", "setuid": false, "setgid": false, "sticky": false, "r": true, "w": true, "x": true, "st_blksize": 4096, "st_blocks": 0, "st_flags": 0, "st_rdev": 0, "st_gen": 0}
+
+As an example, to compare modification time between two files, use an expression like ``f1.stat().st_mtime < f2.stat().st_mtime``.
+
+This is longer than the traditional bash expression, but much more clear.
+
+The file "kind" is a one-letter code:
+:b: block
+:c: character-mode
+:d: directory
+:f: regular file
+:p: FIFO or pipe
+:l: symbolic link
+:s: socket
+
+The ``r``, ``w``, and ``x`` attributes indicate if the current effective userid can read, write, or execute the file. This comes from the detailed permission bits.
+
+The intent is to provide a single, uniform implementation for arithmetic and logic operations.
+The primary use case integration into an DSL-based application to provide expressions without the mental burden of writing the parser and evaluator.
+
+We can also use CEL interactively, because, why not?
+
+.. code-block:: bash
+
+ % python -m celpy -i
+ Enter an expression to have it evaluated.
+ CEL> 355. / 113.
+ 3.1415929203539825
+ CEL> ?
+
+ Documented commands (type help ):
+ ========================================
+ bye exit help quit set show
+
+ CEL> help set
+ Set variable expression
+
+ Evaluates the expression, saves the result as the given variable in the current activation.
+
+ CEL> set a 6
+ 6
+ CEL> set b 7
+ 7
+ CEL> a * b
+ 42
+ CEL> show
+ {'a': IntType(6), 'b': IntType(7)}
+ CEL> bye
+ %
+
+The ``bye``, ``exit``, and ``quit`` commands all exit the application.
diff --git a/docs/build/html/_sources/configuration.rst.txt b/docs/build/html/_sources/configuration.rst.txt
index e029b5d..93bf760 100644
--- a/docs/build/html/_sources/configuration.rst.txt
+++ b/docs/build/html/_sources/configuration.rst.txt
@@ -2,25 +2,13 @@
# Copyright 2020 The Cloud Custodian Authors.
# SPDX-License-Identifier: Apache-2.0
+.. _configuration:
+
######################
Configuration
######################
-The CLI application can bind argument values from the environment.
-The command-line provides variable names and type information.
-The OS environment provides string values.
-
-.. code:: bash
-
- export x=6
- export y=7
- celpy -n --arg x:int --arg y:int 'x*y'
- 42
-
-While this example uses the OS environment,
-it isn't the usual sense of *configuration*.
-The only configuration options available for the command-line application are the logging configuration.
-
+The **celpy** package uses a configuration file to set the logging options.
If a ``celpy.toml`` file exists in the local directory or the user's ``HOME`` directory, this will be used to provide logging configuration for the ``celpy`` application.
This file must have a ``logging`` paragraph.
@@ -40,11 +28,41 @@ This paragraph can contain the parameters for logging configuration.
class = "logging.StreamHandler"
formatter = "console"
+This provides minimal log output, showing only warnings, errors, and fatal error messages.
+The ``root.level`` needs to be "INFO" or "DEBUG" to see more output.
+Setting a specific logger's level to "DEBUG" will raise the logging level for a specific component.
+
+All of the **celpy** loggers have names starting with ``celpy.``.
+This permits integration with other application without polluting those logs with **celpy** output.
+
To enable very detailed debugging, do the following:
- Set the ``CEL_TRACE`` environment variable to some non-empty value, like ``"true"``.
This enables a ``@trace`` decorator on some evaluation methods.
-- In a ``[logging.loggers.celpy.Evaluator]`` paragraph, set ``level = "DEBUG"``.
+- Add a ``[logging.loggers.celpy.Evaluator]`` paragraph, with ``level = "DEBUG"``.
+ This can be done for any of the ``celpy`` components with loggers.
+
+- In the ``[logging]`` paragraph, set ``root.level = "DEBUG"``.
+
+Loggers include the following:
+
+- ``celpy``
+
+- ``celpy.Runner``
+
+- ``celpy.Environment``
+
+- ``celpy.repl``
+
+- ``celpy.c7nlib``
+
+- ``celpy.celtypes``
+
+- ``celpy.evaluation``
+
+- ``celpy.NameContainer``
+
+- ``celpy.Evaluator``
-- Set the ``[logging]`` paragraph, set ``root.level = "DEBUG"``.
+- ``celpy.Transpiler``
diff --git a/docs/build/html/_sources/development.rst.txt b/docs/build/html/_sources/development.rst.txt
new file mode 100644
index 0000000..02a00f3
--- /dev/null
+++ b/docs/build/html/_sources/development.rst.txt
@@ -0,0 +1,288 @@
+.. comment
+ # Copyright 2025 The Cloud Custodian Authors.
+ # SPDX-License-Identifier: Apache-2.0
+
+######################
+Development Tools
+######################
+
+The development effort is dependent on several parts of the CEL project.
+
+1. The language specification. https://github.com/google/cel-spec/blob/master/doc/langdef.md
+
+2. The test cases. https://github.com/google/cel-spec/tree/master/tests/simple/testdata
+
+The language specification is transformed into a Lark grammar.
+This is in the ``src/cel.lark`` file.
+This changes very slowly.
+Any changes must be reflected (manually) by revising the lark version of the EBNF.
+
+The test cases present a more challenging problem.
+
+A tool, ``pb2g.py``, converts the test cases from Protobuf messages to Gherkin scenarios.
+
+.. uml::
+
+ @startuml
+
+ file source as "source protobuf test cases"
+ file features as "Gherkin feature files"
+
+ source --> [pb2g.py]
+ [pb2g.py] --> [Docker] : "Uses"
+ [Docker] ..> [mkgherkin.go] : "Runs"
+ [pb2g.py] --> features
+ @enduml
+
+
+The pb2g Tool
+==============
+
+The ``pb2g.py`` Python application converts a protobuf test case collection into a Gherkin Feature file.
+These can be used to update the ``features`` directory.
+
+SYNOPSIS
+---------
+
+.. program:: python tools/pb2g.py [-g docker|local] [-o output] [-sv] source
+
+.. option:: -g , --gherkinizer , --output