diff --git a/MODULE.bazel b/MODULE.bazel index 4837c20..0f25615 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -33,6 +33,7 @@ git_override( # https://registry.bazel.build/modules/cel-conformance bazel_dep(name = "cel-conformance", version = "0.1.0", repo_name = "cel_conformance") + # TODO(b/582952496): Remove git_override once cel-conformance is published on BCR. git_override( module_name = "cel-conformance", @@ -62,12 +63,33 @@ single_version_override( version = "0.2.18", ) +# https://registry.bazel.build/modules/rules_license +bazel_dep(name = "rules_license", version = "1.0.0") + # https://registry.bazel.build/modules/rules_proto bazel_dep(name = "rules_proto", version = "7.1.0") # https://registry.bazel.build/modules/rules_python bazel_dep(name = "rules_python", version = "1.9.0") +# https://registry.bazel.build/modules/yaml-cpp +bazel_dep(name = "yaml-cpp", version = "0.9.0") + +# To update: copy _CEL_POLICY_TAG and _CEL_POLICY_SHA pick a commit from +# https://github.com/cel-expr/cel-policy +_CEL_POLICY_TAG = "8e4cf0378e3f0d3406b537afbcc6f59562652ad6" + +_CEL_POLICY_SHA = "c3aaed647f04efb7f77d67de21f21720d298938a066cec7f94dd9e24f9a8d88f" + +http_archive = use_repo_rule("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive") + +http_archive( + name = "cel_policy", + sha256 = _CEL_POLICY_SHA, + strip_prefix = "cel-policy-%s" % _CEL_POLICY_TAG, + url = "https://github.com/cel-expr/cel-policy/archive/%s.tar.gz" % _CEL_POLICY_TAG, +) + # Configure rules_python's hermetic toolchains to resolve external # dependencies natively. Under Windows, compiling pybind C++ extensions # against Python 3.11 headers requires that the execution environment @@ -84,6 +106,29 @@ use_repo(python, "pythons_hub") register_toolchains("@pythons_hub//:all") +# PyYAML (pure-Python sources) for the policy conformance test. PyYAML is not +# on BCR; to update, pick a tag from https://github.com/yaml/pyyaml/tags +_PYYAML_VERSION = "6.0.2" + +_PYYAML_SHA = "9377c381ac3fccad8df73d96b5139ef8b1a2c57a0d913e95ab0a2275d66b5caa" + +http_archive( + name = "pyyaml", + build_file_content = """ +load("@rules_python//python:py_library.bzl", "py_library") + +py_library( + name = "yaml", + srcs = glob(["lib/yaml/*.py"]), + imports = ["lib"], + visibility = ["//visibility:public"], +) +""", + sha256 = _PYYAML_SHA, + strip_prefix = "pyyaml-%s" % _PYYAML_VERSION, + url = "https://github.com/yaml/pyyaml/archive/refs/tags/%s.tar.gz" % _PYYAML_VERSION, +) + pybind11_internal_configure = use_extension( "@pybind11_bazel//:internal_configure.bzl", "internal_configure_extension", diff --git a/README.md b/README.md index 29c923a..869629f 100644 --- a/README.md +++ b/README.md @@ -314,6 +314,25 @@ env = cel.NewEnv(pool, extensions=[ext_math.ExtMath()]) expr = env.compile("math.sqrt(4)") ``` +#### Working with optional values + +When `ext_optional.ExtOptional()` is enabled, CEL `optional_type` values map to +`cel.Optional` (`cel.Optional.of(value)` and `cel.Optional.none()`), and +optional variable/function types can be declared with `cel.Type.Optional(...)` +(or `cel.Type.OPTIONAL` for `optional_type`): + +```python +env = cel.NewEnv( + variables={"x": cel.Type.Optional(cel.Type.INT)}, + extensions=[ext_optional.ExtOptional()], +) +res = env.compile("x").eval(data={"x": cel.Optional.of(42)}) +opt = res.plain_value() # cel.Optional.of(42) +print(opt.has_value()) # True +print(opt.value()) # 42 +print(opt.or_value(0)) # 42 +``` + #### Defining a custom extension in Python You can define custom functions and pass them as an extension. diff --git a/cel_expr_python/BUILD b/cel_expr_python/BUILD index 02c8f9c..9300006 100644 --- a/cel_expr_python/BUILD +++ b/cel_expr_python/BUILD @@ -1,5 +1,6 @@ load("@pybind11_bazel//:build_defs.bzl", "pybind_extension", "pybind_library") load("@rules_cc//cc:cc_library.bzl", "cc_library") +load("@rules_python//python:py_library.bzl", "py_library") load("@rules_python//python:py_test.bzl", "py_test") package(default_visibility = ["//visibility:public"]) @@ -20,6 +21,7 @@ pybind_library( "py_cel_expression.cc", "py_cel_function.cc", "py_cel_function_decl.cc", + "py_cel_optional.cc", "py_cel_options.cc", "py_cel_overload.cc", "py_cel_python_extension.cc", @@ -38,6 +40,7 @@ pybind_library( "py_cel_expression.h", "py_cel_function.h", "py_cel_function_decl.h", + "py_cel_optional.h", "py_cel_options.h", "py_cel_overload.h", "py_cel_python_extension.h", @@ -50,6 +53,7 @@ pybind_library( tags = ["freethreading_compatible"], visibility = [":__subpackages__"], deps = [ + ":cel_compiler_capsule", ":cel_extension", ":status_macros", "@com_google_absl//absl/base", @@ -142,6 +146,18 @@ pybind_library( ], ) +# Shares the `cel::Compiler` owned by a `cel.Env` with separately loaded +# pybind11 modules (e.g. `cel_policy`). +pybind_library( + name = "cel_compiler_capsule", + hdrs = ["cel_compiler_capsule.h"], + tags = ["freethreading_compatible"], + deps = [ + ":cel_extension", + "@com_google_cel_cpp//compiler", + ], +) + cc_library( name = "status_macros", hdrs = ["status_macros.h"], @@ -150,20 +166,34 @@ cc_library( ], ) +py_library( + name = "leak_check", + testonly = True, + srcs = ["leak_check.py"], + visibility = [":__subpackages__"], +) + py_test( name = "cel_test", srcs = ["cel_test.py"], data = [ ":cel", + "//cel_expr_python/ext:ext_optional", ], deps = [ + ":leak_check", "//testing:proto2_test_all_types_py_pb2", "@com_google_absl_py//absl/testing:absltest", + "@com_google_absl_py//absl/testing:parameterized", + "@com_google_cel_spec//proto/cel/expr:checked_py_pb2", "@com_google_protobuf//:protobuf", "@com_google_protobuf//:protobuf_python", ] + select({ "@platforms//os:windows": [], - "//conditions:default": [":cel"], + "//conditions:default": [ + ":cel", + "//cel_expr_python/ext:ext_optional", + ], }), ) @@ -174,6 +204,7 @@ py_test( ":cel", ], deps = [ + ":leak_check", "//testing:proto2_test_all_types_py_pb2", "@com_google_absl_py//absl/testing:absltest", ] + select({ @@ -196,6 +227,7 @@ py_test( deps = [ "//testing:proto2_test_all_types_py_pb2", "@com_google_absl_py//absl/testing:absltest", + "@com_google_absl_py//absl/testing:parameterized", ] + select({ "@platforms//os:windows": [], "//conditions:default": [ diff --git a/cel_expr_python/cel.pyi b/cel_expr_python/cel.pyi index 8695ec4..fa59b46 100644 --- a/cel_expr_python/cel.pyi +++ b/cel_expr_python/cel.pyi @@ -1,9 +1,12 @@ from collections.abc import Callable, Mapping, Sequence import datetime -from typing import Any, ClassVar, TypeAlias, Union +from typing import Any, ClassVar, Generic, TypeAlias, TypeVar, Union from google.protobuf import descriptor_pool as proto_descriptor_pool from google.protobuf import message as proto_message +_T = TypeVar('_T') +_D = TypeVar('_D') + class Activation: pass class CelExtension(CelExtensionBase): @@ -46,6 +49,18 @@ class FunctionDecl: class Overload: def __init__(self, id: str | None = ..., return_type: Type = ..., parameters: Sequence[Type] = ..., is_member: bool = ..., impl: Callable[..., Any] = ..., signature: str | None = ...) -> None: ... +class Optional(Generic[_T]): + def __init__(self) -> None: ... + @staticmethod + def of(value: _T) -> Optional[_T]: ... + @staticmethod + def none() -> Optional[Any]: ... + def has_value(self) -> bool: ... + def value(self) -> _T: ... + def or_value(self, default: _D) -> _T | _D: ... + def __bool__(self) -> bool: ... + def __eq__(self, other: object) -> bool: ... + class Type: BOOL: ClassVar[Type] = ... BYTES: ClassVar[Type] = ... @@ -57,6 +72,7 @@ class Type: LIST: ClassVar[Type] = ... MAP: ClassVar[Type] = ... NULL: ClassVar[Type] = ... + OPTIONAL: ClassVar[Type] = ... STRING: ClassVar[Type] = ... TIMESTAMP: ClassVar[Type] = ... TYPE: ClassVar[Type] = ... @@ -70,6 +86,8 @@ class Type: @staticmethod def Map(key_type: Type, element_type: Type) -> Type: ... @staticmethod + def Optional(element_type: Type = ...) -> Type: ... + @staticmethod def Type(element_type: Type) -> Type: ... def is_assignable_from(self, arg0: Type) -> bool: ... def is_message(self) -> bool: ... @@ -77,8 +95,8 @@ class Type: def __eq__(self, arg0: object) -> bool: ... BaseValue: TypeAlias = Union[None, bool, int, float, str, bytes, proto_message.Message, datetime.datetime, datetime.timedelta, Type] -PlainValue: TypeAlias = Union[BaseValue, list[PlainValue], dict[PlainValue, PlainValue]] -CelValue: TypeAlias = Union[BaseValue, list[Value], dict[PlainValue, Value]] +PlainValue: TypeAlias = Union[BaseValue, Optional[PlainValue], list[PlainValue], dict[PlainValue, PlainValue]] +CelValue: TypeAlias = Union[BaseValue, Optional[Value], list[Value], dict[PlainValue, Value]] class Value: def equals(self, other: Value) -> bool: ... diff --git a/cel_expr_python/cel_compiler_capsule.h b/cel_expr_python/cel_compiler_capsule.h new file mode 100644 index 0000000..5504cc2 --- /dev/null +++ b/cel_expr_python/cel_compiler_capsule.h @@ -0,0 +1,86 @@ +/* + * Copyright 2026 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Shares the `cel::Compiler` owned by a `cel.Env` with pybind11 modules that +// are built and loaded separately from the `cel` module (e.g. `cel_policy`) +// without linking against the `cel` module's internals. +// +// The `cel` module exposes the compiler via the private +// `cel.Env._compiler_capsule()` method, which returns a PyCapsule created by +// `MakeCelCompilerCapsule()`. Other modules read it with `GetCelCompiler()`. + +#ifndef THIRD_PARTY_CEL_PYTHON_CEL_COMPILER_CAPSULE_H_ +#define THIRD_PARTY_CEL_PYTHON_CEL_COMPILER_CAPSULE_H_ + +#include // IWYU pragma: keep - Needed for PyCapsule_GetPointer + +#include +#include + +#include "compiler/compiler.h" +#include "cel_expr_python/cel_extension.h" +#include + +namespace cel_python { + +// Name of the PyCapsule returned by `cel.Env._compiler_capsule()`. The capsule +// holds a heap-allocated `std::shared_ptr`, which keeps +// the environment that owns the compiler alive for the capsule's lifetime. +inline constexpr char kCelCompilerCapsuleName[] = + "cel_python.CompilerSharedPtr"; + +// Wraps `compiler` in a PyCapsule named `kCelCompilerCapsuleName`. The capsule +// shares ownership of `compiler` and releases it when garbage collected. +inline pybind11::capsule MakeCelCompilerCapsule( + std::shared_ptr compiler) { + return pybind11::capsule( + new std::shared_ptr(std::move(compiler)), + kCelCompilerCapsuleName, [](void* ptr) { + delete static_cast*>(ptr); + }); +} + +// Returns the compiler owned by `env`, which must be a `cel.Env` instance. +// The returned pointer shares ownership of the compiler, so it remains valid +// even if `env` is garbage collected. Requires the GIL. +// +// Throws `TypeError` if `env` is not a `cel.Env`. +inline std::shared_ptr GetCelCompiler( + pybind11::handle env) { + PYBIND11_CONSTINIT static pybind11::gil_safe_call_once_and_store< + pybind11::object> + env_type_storage; + const pybind11::object& env_type = + env_type_storage + .call_once_and_store_result([] { + return pybind11::module_::import(CEL_MODULE_NAME).attr("Env"); + }) + .get_stored(); + if (!pybind11::isinstance(env, env_type)) { + throw pybind11::type_error("env must be an instance of cel.Env"); + } + pybind11::capsule capsule = + env.attr("_compiler_capsule")().cast(); + void* ptr = PyCapsule_GetPointer(capsule.ptr(), kCelCompilerCapsuleName); + if (ptr == nullptr) { + throw pybind11::error_already_set(); + } + return *static_cast*>(ptr); +} + +} // namespace cel_python + +#endif // THIRD_PARTY_CEL_PYTHON_CEL_COMPILER_CAPSULE_H_ diff --git a/cel_expr_python/cel_env_test.py b/cel_expr_python/cel_env_test.py index 7684169..894d681 100644 --- a/cel_expr_python/cel_env_test.py +++ b/cel_expr_python/cel_env_test.py @@ -18,10 +18,12 @@ ability to be created from and serialized to YAML format. """ +import datetime import textwrap from typing import Any from absl.testing import absltest +from absl.testing import parameterized from cel.expr.conformance.proto2 import test_all_types_pb2 as test_all_types_pb from cel_expr_python import cel from cel_expr_python.ext import ext_bindings @@ -30,7 +32,7 @@ from cel_expr_python.ext import ext_strings -class CelEnvTest(absltest.TestCase): +class CelEnvTest(parameterized.TestCase): def test_env_config_from_and_to_yaml(self): config: cel.EnvConfig = cel.NewEnvConfigFromYaml(""" @@ -304,6 +306,7 @@ def test_config_export_variables(self): "var_int_map": cel.Type.Map(cel.Type.INT, cel.Type.STRING), "var_string_map": cel.Type.Map(cel.Type.STRING, cel.Type.BOOL), "var_dyn_map": cel.Type.MAP, + "var_opt_int": cel.Type.Optional(cel.Type.INT), "var_dyn": cel.Type.DYN, } ) @@ -332,6 +335,8 @@ def test_config_export_variables(self): type: "map" - name: "var_msg" type: "cel.expr.conformance.proto2.TestAllTypes" + - name: "var_opt_int" + type: "optional_type" - name: "var_str" type: "string" - name: "var_string_list" @@ -743,6 +748,622 @@ def test_env_options(self): default_env = cel.NewEnv() self.assertTrue(default_env.options().enable_pratt_parser) + def _new_poly_env(self) -> cel.Env: + ext = cel.CelExtension( + "poly_ext", + functions=[ + cel.FunctionDecl( + "ext_id", + [ + cel.Overload( + signature="ext_id(~T)", + return_type=cel.Type("~T"), + impl=lambda x: x, + ) + ], + ) + ], + ) + return cel.NewEnv( + extensions=[ext, ext_optional.ExtOptional()], + variables={ + "var_tp": cel.Type("~T"), + "var_type": cel.Type.TYPE, + "var_int_type": cel.Type.Type(cel.Type.INT), + "var_dyn": cel.Type.DYN, + }, + functions=[ + cel.FunctionDecl( + "to_string", + [ + cel.Overload( + signature="to_string(dyn)", + return_type=cel.Type.STRING, + impl=repr, + ) + ], + ), + cel.FunctionDecl( + "dyn_id", + [ + cel.Overload( + signature="dyn_id(dyn)", + return_type=cel.Type.DYN, + impl=lambda x: x, + ) + ], + ), + cel.FunctionDecl( + "identity", + [ + cel.Overload( + signature="identity(~T)", + return_type=cel.Type("~T"), + impl=lambda x: x, + ) + ], + ), + cel.FunctionDecl( + "tp_id", + [ + cel.Overload( + signature="tp_id(~T)", + return_type=cel.Type("~T"), + impl=lambda x: x, + ) + ], + ), + cel.FunctionDecl( + "bad_type_ret", + [ + cel.Overload( + signature="bad_type_ret()", + return_type=cel.Type.TYPE, + impl=lambda: "not_a_type", + ) + ], + ), + cel.FunctionDecl( + "make_tp", + [ + cel.Overload( + signature="make_tp()", + return_type=cel.Type("~T"), + ) + ], + ), + cel.FunctionDecl( + "late_ovl", + [ + cel.Overload( + signature="late_ovl(int)", + return_type=cel.Type.STRING, + ), + cel.Overload( + signature="late_ovl(string)", + return_type=cel.Type.STRING, + ), + ], + ), + cel.FunctionDecl( + "late_dyn", + [ + cel.Overload( + signature="late_dyn(dyn)", + return_type=cel.Type.STRING, + ), + ], + ), + cel.FunctionDecl( + "late_tp", + [ + cel.Overload( + signature="late_tp(~T)", + return_type=cel.Type("~T"), + ), + ], + ), + ], + ) + + def test_functions_with_dyn_and_type_params(self): + env = self._new_poly_env() + self.assertEqual(env.compile("to_string(42)").eval().value(), "42") + self.assertEqual(env.compile("identity('abc')").eval().value(), "abc") + self.assertEqual(env.compile("ext_id(7) + 1").eval().value(), 8) + + # Type-parameterized variables, parameterized/abstract TypeValues, invalid + # TYPE conversions, and overloaded activation functions. + self.assertEqual(env.compile("var_tp").return_type(), cel.Type.DYN) + self.assertEqual(env.compile("make_tp()").return_type(), cel.Type.DYN) + self.assertEqual( + env.compile("tp_id(var_type)") + .eval(data={"var_type": cel.Type.Type(cel.Type.INT)}) + .value(), + cel.Type.Type(cel.Type.INT), + ) + self.assertEqual( + env.compile("tp_id(var_int_type)") + .eval(data={"var_int_type": cel.Type.INT}) + .value(), + cel.Type.INT, + ) + abstract_type = cel.Type.AbstractType("optional_type", [cel.Type.INT]) + self.assertEqual( + env.compile("tp_id(var_dyn)") + .eval(data={"var_dyn": abstract_type}) + .value(), + abstract_type, + ) + coll_type = cel.Type.AbstractType("collection", [cel.Type.DOUBLE]) + self.assertEqual( + env.compile("tp_id(var_type)") + .eval(data={"var_type": coll_type}) + .value(), + coll_type, + ) + self.assertEqual( + env.compile("tp_id(var_type)") + .eval(data={"var_type": cel.Type.Type(coll_type)}) + .value(), + cel.Type.Type(coll_type), + ) + bad_type_var_res = env.compile("var_type").eval( + data={"var_type": "not_a_type"} + ) + self.assertEqual(bad_type_var_res.type(), cel.Type.ERROR) + self.assertIn( + "Unexpected value type for 'var_type': str. (Expected TYPE)", + bad_type_var_res.value(), + ) + bad_type_fn_res = env.compile("bad_type_ret()").eval() + self.assertEqual(bad_type_fn_res.type(), cel.Type.ERROR) + self.assertIn("(Expected TYPE)", bad_type_fn_res.value()) + + ovl_act = env.Activation( + {}, + functions=[ + cel.Function( + "late_ovl", + [cel.Type.INT], + False, + lambda x: f"int:{x}", + return_type=cel.Type.STRING, + ), + cel.Function( + "late_ovl", + [cel.Type.STRING], + False, + lambda x: f"str:{x}", + return_type=cel.Type.STRING, + ), + ], + ) + self.assertEqual(env.compile("late_ovl(1)").eval(ovl_act).value(), "int:1") + self.assertEqual( + env.compile("late_ovl('a')").eval(ovl_act).value(), "str:a" + ) + + poly_act = env.Activation( + {}, + functions=[ + cel.Function( + "late_dyn", + [cel.Type.DYN], + False, + repr, + return_type=cel.Type.STRING, + ), + cel.Function( + "late_tp", + [cel.Type("~T")], + False, + lambda x: x, + return_type=cel.Type("~T"), + ), + ], + ) + self.assertEqual(env.compile("late_dyn(42)").eval(poly_act).value(), "42") + self.assertEqual( + env.compile("late_dyn('abc')").eval(poly_act).value(), "'abc'" + ) + self.assertEqual(env.compile("late_tp(7)").eval(poly_act).value(), 7) + self.assertEqual( + env.compile("late_tp('abc')").eval(poly_act).value(), "abc" + ) + + @parameterized.named_parameters( + ("null_eq", "ext_id(null) == null", True), + ("bool_not", "!ext_id(false)", True), + ("int_add", "identity(7) + 1", 8), + ("uint_add", "ext_id(9223372036854775808u) + 1u", 9223372036854775809), + ("double_add", "identity(1.5) + 2.5", 4.0), + ("string_add", "ext_id('hello') + '!'", "hello!"), + ("bytes_add", "identity(b'ab') + b'cd'", bytearray(b"abcd")), + ("list_add", "ext_id([1, 2]) + [3]", [1, 2, 3]), + ("map_index", "identity({'a': 1})['a']", 1), + ("type_eq", "ext_id(type(1)) == int", True), + ("type_list_eq", "ext_id(type([1])) == list", True), + ("type_map_eq", "ext_id(type({'a': 1})) == map", True), + ( + "type_ts_eq", + ( + "ext_id(type(timestamp('2025-01-01T00:00:00Z'))) ==" + " google.protobuf.Timestamp" + ), + True, + ), + ( + "type_dur_eq", + "ext_id(type(duration('1s'))) == google.protobuf.Duration", + True, + ), + ( + "timestamp_add", + "identity(timestamp('2025-01-01T00:00:00Z')) + duration('1h')", + datetime.datetime(2025, 1, 1, 1, 0, tzinfo=datetime.timezone.utc), + ), + ( + "duration_add", + "ext_id(duration('10s')) + duration('5s')", + datetime.timedelta(seconds=15), + ), + ( + "proto_field", + ( + "ext_id(cel.expr.conformance.proto2.TestAllTypes" + "{single_string: 'hello'}).single_string" + ), + "hello", + ), + ( + "wrapper_add", + "ext_id(google.protobuf.StringValue{value: 'wrap'}) + 'ped'", + "wrapped", + ), + ) + def test_type_param_function_in_expressions( + self, + expr_str: str, + expected_val: Any, + ): + env = self._new_poly_env() + res = env.compile(expr_str).eval() + self.assertEqual(res.value(), expected_val) + + @parameterized.named_parameters( + ("null", "null", None, cel.Type.NULL), + ("bool_true", "true", True, cel.Type.BOOL), + ("bool_false", "false", False, cel.Type.BOOL), + ("int", "-42", -42, cel.Type.INT), + ( + "uint_large", + "18446744073709551615u", + 18446744073709551615, + cel.Type.UINT, + ), + ("double", "3.14", 3.14, cel.Type.DOUBLE), + ("string", "'abc'", "abc", cel.Type.STRING), + ("bytes", r"b'ab\x00c'", bytearray(b"ab\x00c"), cel.Type.BYTES), + ("list", "[1, 'two', true]", [1, "two", True], cel.Type.LIST), + ("map", "{'a': 1, 'b': 'two'}", {"a": 1, "b": "two"}, cel.Type.MAP), + ( + "timestamp", + "timestamp('2025-06-15T12:30:00.500000Z')", + datetime.datetime( + 2025, 6, 15, 12, 30, 0, 500000, datetime.timezone.utc + ), + cel.Type.TIMESTAMP, + ), + ( + "duration", + "duration('10s')", + datetime.timedelta(seconds=10), + cel.Type.DURATION, + ), + ("type_int", "type(1)", cel.Type.INT, cel.Type.TYPE), + ("type_null", "type(null)", cel.Type.NULL, cel.Type.TYPE), + ("type_type", "type(int)", cel.Type.TYPE, cel.Type.TYPE), + ( + "type_msg", + "type(cel.expr.conformance.proto2.TestAllTypes{})", + cel.Type("cel.expr.conformance.proto2.TestAllTypes"), + cel.Type.TYPE, + ), + ( + "proto_msg", + ( + "cel.expr.conformance.proto2.TestAllTypes" + "{single_string: 'hello', single_int32: 7}" + ), + test_all_types_pb.TestAllTypes(single_string="hello", single_int32=7), + cel.Type("cel.expr.conformance.proto2.TestAllTypes"), + ), + ( + "wkt_bool_value", + "google.protobuf.BoolValue{value: true}", + True, + cel.Type.BOOL, + ), + ( + "wkt_int32_value", + "google.protobuf.Int32Value{value: -12}", + -12, + cel.Type.INT, + ), + ( + "wkt_int64_value", + "google.protobuf.Int64Value{value: 42}", + 42, + cel.Type.INT, + ), + ( + "wkt_uint64_value", + "google.protobuf.UInt64Value{value: 18446744073709551615u}", + 18446744073709551615, + cel.Type.UINT, + ), + ( + "wkt_float_value", + "google.protobuf.FloatValue{value: 1.5}", + 1.5, + cel.Type.DOUBLE, + ), + ( + "wkt_double_value", + "google.protobuf.DoubleValue{value: 2.5}", + 2.5, + cel.Type.DOUBLE, + ), + ( + "wkt_string_value", + "google.protobuf.StringValue{value: 'wrapped'}", + "wrapped", + cel.Type.STRING, + ), + ( + "wkt_bytes_value", + "google.protobuf.BytesValue{value: b'bin'}", + bytearray(b"bin"), + cel.Type.BYTES, + ), + ( + "wkt_empty_wrapper", + "google.protobuf.Int64Value{}", + 0, + cel.Type.INT, + ), + ( + "wkt_unset_wrapper_field", + "cel.expr.conformance.proto2.TestAllTypes{}.single_int64_wrapper", + None, + cel.Type.NULL, + ), + ( + "wkt_empty_value", + "google.protobuf.Value{}", + None, + cel.Type.NULL, + ), + ( + "wkt_struct", + ( + "google.protobuf.Struct" + "{fields: {'k': google.protobuf.Value{string_value: 'v'}}}" + ), + {"k": "v"}, + cel.Type.MAP, + ), + ( + "wkt_list_value", + ( + "google.protobuf.ListValue" + "{values: [google.protobuf.Value{number_value: 2.0}]}" + ), + [2.0], + cel.Type.LIST, + ), + ( + "wkt_value", + "google.protobuf.Value{bool_value: true}", + True, + cel.Type.BOOL, + ), + ( + "wkt_any", + ( + "cel.expr.conformance.proto2.TestAllTypes" + "{single_any: cel.expr.conformance.proto2.TestAllTypes" + "{single_string: 'packed'}}.single_any" + ), + test_all_types_pb.TestAllTypes(single_string="packed"), + cel.Type("cel.expr.conformance.proto2.TestAllTypes"), + ), + ( + "nested_list", + "[timestamp('2025-06-15T12:30:00.500000Z'), duration('5s'), type(1)]", + [ + datetime.datetime( + 2025, 6, 15, 12, 30, 0, 500000, datetime.timezone.utc + ), + datetime.timedelta(seconds=5), + cel.Type.INT, + ], + cel.Type.LIST, + ), + ( + "nested_map", + "{1: duration('2s'), 2: type('a')}", + {1: datetime.timedelta(seconds=2), 2: cel.Type.STRING}, + cel.Type.MAP, + ), + ) + def test_dyn_function_values( + self, + expr_str: str, + expected_val: Any, + expected_type: cel.Type, + ): + env = self._new_poly_env() + res = env.compile(f"dyn_id({expr_str})").eval() + self.assertEqual(res.type(), expected_type) + self.assertEqual(res.value(), expected_val) + self.assertEqual(res.plain_value(), expected_val) + + @parameterized.named_parameters( + ( + "list", + "type([1])", + "list", + cel.Type.Type(cel.Type.List(cel.Type.INT)), + cel.Type.LIST, + ), + ( + "list_cross_elem", + "type([1])", + "type(['a'])", + cel.Type.Type(cel.Type.List(cel.Type.INT)), + cel.Type.LIST, + ), + ( + "map", + "type({'a': 1})", + "map", + cel.Type.Type(cel.Type.Map(cel.Type.STRING, cel.Type.INT)), + cel.Type.MAP, + ), + ( + "type_int", + "type(int)", + "type", + cel.Type.Type(cel.Type.Type(cel.Type.INT)), + cel.Type.TYPE, + ), + ( + "type_type_1", + "type(type(1))", + "type", + cel.Type.Type(cel.Type.Type(cel.Type.INT)), + cel.Type.TYPE, + ), + ( + "optional_of", + "type(optional.of(1))", + "optional_type", + cel.Type.Type(cel.Type.AbstractType("optional_type", [cel.Type.INT])), + cel.Type.AbstractType("optional_type", [cel.Type.DYN]), + ), + ( + "optional_of_cross_elem", + "type(optional.of(1))", + "type(optional.of('a'))", + cel.Type.Type(cel.Type.AbstractType("optional_type", [cel.Type.INT])), + cel.Type.AbstractType("optional_type", [cel.Type.DYN]), + ), + ( + "optional_none", + "type(optional.none())", + "optional_type", + cel.Type.Type(cel.Type.AbstractType("optional_type", [cel.Type.DYN])), + cel.Type.AbstractType("optional_type", [cel.Type.DYN]), + ), + ) + def test_runtime_type_erasure( + self, + type_expr: str, + expected_cel_type: str, + expected_static_type: cel.Type, + expected_runtime_val: cel.Type, + ): + env = self._new_poly_env() + expr = env.compile(type_expr) + self.assertEqual(expr.return_type(), expected_static_type) + self.assertEqual(expr.eval().value(), expected_runtime_val) + self.assertTrue( + env.compile(f"{type_expr} == {expected_cel_type}").eval().value() + ) + tp_expr = env.compile(f"tp_id({type_expr})") + self.assertEqual(tp_expr.return_type(), expected_static_type) + self.assertEqual(tp_expr.eval().value(), expected_runtime_val) + self.assertTrue( + env.compile(f"tp_id({type_expr}) == {expected_cel_type}").eval().value() + ) + + def test_abstract_type_config_and_value_type(self): + env = cel.NewEnv( + variables={ + "var_coll": cel.Type.AbstractType("collection", [cel.Type.DOUBLE]) + }, + extensions=[ext_optional.ExtOptional(1)], + ) + self.assertEqual( + normalize_yaml(env.config().to_yaml()), + normalize_yaml(""" + extensions: + - name: "optional" + version: 1 + variables: + - name: "var_coll" + type: "collection" + """), + ) + res = env.compile("optional.none()").eval() + self.assertEqual( + res.type(), cel.Type.AbstractType("optional_type", [cel.Type.DYN]) + ) + coll_ext = cel.CelExtension( + "coll_ext", + functions=[ + cel.FunctionDecl( + "coll_id", + [ + cel.Overload( + "coll_id_ovl", + cel.Type.AbstractType("collection", [cel.Type.DOUBLE]), + [ + cel.Type.AbstractType( + "collection", [cel.Type.DOUBLE] + ) + ], + impl=lambda x: x, + ) + ], + ) + ], + ) + coll_env = cel.NewEnv( + variables={ + "var_coll": cel.Type.AbstractType("collection", [cel.Type.DOUBLE]) + }, + extensions=[coll_ext], + ) + self.assertEqual( + coll_env.compile("coll_id(var_coll)").return_type(), + cel.Type.AbstractType("collection", [cel.Type.DOUBLE]), + ) + + def test_config_function_dyn_parameter(self): + config: cel.EnvConfig = cel.NewEnvConfigFromYaml(""" + functions: + - name: is_map_with_key + overloads: + - signature: "is_map_with_key(dyn)" + return: "bool" + """) + env: cel.Env = cel.NewEnv( + config=config, + variables={"val": cel.Type.Map(cel.Type.STRING, cel.Type.DYN)}, + function_impls={ + "is_map_with_key(dyn)": ( + lambda arg: isinstance(arg, dict) and "k" in arg + ), + }, + ) + expr = env.compile("is_map_with_key(val)") + self.assertTrue(expr.eval(data={"val": {"k": 1}}).value()) + self.assertFalse(expr.eval(data={"val": {"other": 1}}).value()) + self.assertFalse(env.compile("is_map_with_key(123)").eval().value()) + class TestCelExtension(cel.CelExtension): """An example CEL extension for testing.""" @@ -755,11 +1376,8 @@ def __init__(self): "hello", [ cel.Overload( - "hello(string)", + signature="hello(string)", return_type=cel.Type.STRING, - parameters=[ - cel.Type.STRING, - ], impl=lambda arg: f"Hello, {arg}!", ) ], diff --git a/cel_expr_python/cel_parallel_test.py b/cel_expr_python/cel_parallel_test.py index 99100fd..8751687 100644 --- a/cel_expr_python/cel_parallel_test.py +++ b/cel_expr_python/cel_parallel_test.py @@ -19,12 +19,14 @@ import dataclasses import gc import logging +import threading import time from typing import Any from absl.testing import absltest from cel.expr.conformance.proto2 import test_all_types_pb2 as test_all_types_pb from cel_expr_python import cel +from cel_expr_python import leak_check @dataclasses.dataclass(frozen=True) @@ -94,6 +96,7 @@ class _TestCase: ] +@leak_check.leak_checked class CelParallelTest(absltest.TestCase): def setUp(self): @@ -119,7 +122,6 @@ def setUp(self): ) ], ) - self.object_counts_before_test = self._grab_object_counts() def tearDown(self): """Tears down the test environment.""" @@ -128,26 +130,6 @@ def tearDown(self): gc.collect() # Assert that all Arenas have been garbage-collected self.assertEqual(cel._InternalArena._get_instance_count(), 0) - self._check_for_leaks() - - def _grab_object_counts(self) -> dict[str, int]: - gc.collect() - all_objects = gc.get_objects() - type_counts = {} - for obj in all_objects: - obj_type = type(obj) - type_counts[obj_type.__name__] = type_counts.get(obj_type, 0) + 1 - return type_counts - - def _check_for_leaks(self): - type_counts = self._grab_object_counts() - for key, count in type_counts.items(): - if count != self.object_counts_before_test.get(key, 0): - self.fail( - f"Object count for {key} did not match expected count. " - f"Expected: {self.object_counts_before_test.get(key, 0)}, " - f"Actual: {count}", - ) def _test_eval(self, multi_threaded: bool): compiled_exprs = [self.env.compile(tc.expr) for tc in _TEST_CASES] @@ -277,6 +259,67 @@ def eval_custom_fn(n: int): results = list(executor.map(eval_custom_fn, range(100))) self.assertLen(results, 100) + def testConcurrentTypeToCelTypeWithDescriptorPoolLookup(self): + """Tests that PyCelType::ToCelType releases the GIL before DescriptorPool.""" + for _ in range(5): + lookup_started = threading.Event() + + class _SlowDescriptorPool: + + def __init__(self, pool: Any, event: threading.Event): + self._pool = pool + self._event = event + + def FindFileByName(self, filename: str): # pylint: disable=invalid-name + return self._pool.FindFileByName(filename) + + def FindFileContainingSymbol(self, symbol_name: str): # pylint: disable=invalid-name + if symbol_name == "cel.expr.conformance.proto2.TestAllTypes": + self._event.set() + # Releases the GIL while DescriptorPool's internal mutex is held. + time.sleep(0.1) + return self._pool.FindFileContainingSymbol(symbol_name) + + slow_pool = _SlowDescriptorPool( + test_all_types_pb.DESCRIPTOR.pool, lookup_started + ) + env = cel.NewEnv( + descriptor_pool=slow_pool, + variables={"var_type": cel.Type.TYPE}, + ) + type_expr = env.compile("var_type == var_type") + # Warm up the runtime/program so eval_type reaches PyObjectToCelValue + # immediately without paying first-time Runtime initialization cost. + self.assertTrue(type_expr.eval(data={"var_type": cel.Type.INT}).value()) + + def compile_message(): + msg_expr = env.compile("cel.expr.conformance.proto2.TestAllTypes{}") + self.assertIsNotNone(msg_expr) + + def eval_type(): + self.assertTrue(lookup_started.wait(timeout=5.0)) + res = type_expr.eval( + data={ + "var_type": cel.Type.List( + cel.Type("cel.expr.conformance.proto2.TestAllTypes") + ) + } + ) + self.assertTrue(res.value()) + + executor = concurrent.futures.ThreadPoolExecutor(max_workers=2) + timed_out = False + try: + compile_future = executor.submit(compile_message) + eval_future = executor.submit(eval_type) + compile_future.result(timeout=5.0) + eval_future.result(timeout=5.0) + except BaseException: + timed_out = True + raise + finally: + executor.shutdown(wait=not timed_out) + if __name__ == "__main__": absltest.main() diff --git a/cel_expr_python/cel_test.py b/cel_expr_python/cel_test.py index 2aca53c..2279e96 100644 --- a/cel_expr_python/cel_test.py +++ b/cel_expr_python/cel_test.py @@ -23,15 +23,21 @@ from google.protobuf import message_factory +from cel.expr import checked_pb2 as checked_pb +from google.protobuf import any_pb2 as any_pb from google.protobuf import duration_pb2 as duration_pb from google.protobuf import timestamp_pb2 as timestamp_pb from absl.testing import absltest +from absl.testing import parameterized from cel.expr.conformance.proto2 import test_all_types_pb2 as test_all_types_pb from cel_expr_python import cel +from cel_expr_python import leak_check +from cel_expr_python.ext import ext_optional +@leak_check.leak_checked @absltest.skipThisClass("Base class") -class _CelTestBase(absltest.TestCase): +class _CelTestBase(parameterized.TestCase): options: cel.Options = cel.Options() def setUp(self): @@ -53,11 +59,12 @@ def setUp(self): "var_int_map": cel.Type.Map(cel.Type.INT, cel.Type.STRING), "var_string_map": cel.Type.Map(cel.Type.STRING, cel.Type.BOOL), "var_dyn_map": cel.Type.MAP, + "var_opt_int": cel.Type.Optional(cel.Type.INT), "var_dyn": cel.Type.DYN, }, + extensions=[ext_optional.ExtOptional()], options=self.options, ) - self.object_counts_before_test = self._grab_object_counts() def tearDown(self): """Tears down the test environment.""" @@ -66,26 +73,6 @@ def tearDown(self): gc.collect() # Assert that all Arenas have been garbage-collected self.assertEqual(cel._InternalArena._get_instance_count(), 0) - self._check_for_leaks() - - def _grab_object_counts(self) -> dict[str, int]: - gc.collect() - all_objects = gc.get_objects() - type_counts = {} - for obj in all_objects: - obj_type = type(obj) - type_counts[obj_type.__name__] = type_counts.get(obj_type, 0) + 1 - return type_counts - - def _check_for_leaks(self): - type_counts = self._grab_object_counts() - for key, count in type_counts.items(): - if count != self.object_counts_before_test.get(key, 0): - self.fail( - f"Object count for {key} did not match expected count. " - f"Expected: {self.object_counts_before_test.get(key, 0)}, " - f"Actual: {count}", - ) def _eval( self, @@ -364,7 +351,52 @@ def testProto_unexpectedType(self): r" .*. \(Expected cel.expr.conformance.proto2.TestAllTypes\)", ) - def testProto_cannotFindMessageClass(self): + @parameterized.named_parameters( + ("message", "TestAllTypes{single_string: 'hello'}", True), + ("list", "[TestAllTypes{single_string: 'hello'}]", True), + ("nested_list", "[[1, 2], TestAllTypes{single_string: 'hello'}]", True), + ("map_value", "{'k': TestAllTypes{single_string: 'hello'}}", True), + ( + "nested_map_value", + ( + "{'a': [1, 2], 'b': [3, 4], 'z':" + " TestAllTypes{single_string: 'hello'}}" + ), + True, + ), + ("optional", "optional.of(TestAllTypes{single_string: 'hello'})", True), + ("call_message", "show(TestAllTypes{single_string: 'hello'})", False), + ("call_list", "show([TestAllTypes{single_string: 'hello'}])", False), + ( + "call_nested_list", + "show([[1, 2], TestAllTypes{single_string: 'hello'}])", + False, + ), + ( + "call_map_value", + "show({'k': TestAllTypes{single_string: 'hello'}})", + False, + ), + ( + "call_nested_map_value", + ( + "show({'a': [1, 2], 'b': [3, 4], 'z':" + " TestAllTypes{single_string: 'hello'}})" + ), + False, + ), + ( + "call_second_arg", + "show([1, 2], TestAllTypes{single_string: 'hello'})", + False, + ), + ( + "call_optional", + "show(optional.of(TestAllTypes{single_string: 'hello'}))", + False, + ), + ) + def testProto_cannotFindMessageClass(self, expr: str, expect_exception: bool): orig_get_message_class = message_factory.GetMessageClass def failing_get_message_class(descriptor): @@ -375,21 +407,110 @@ def failing_get_message_class(descriptor): try: message_factory.GetMessageClass = failing_get_message_class - env = cel.NewEnv(options=self.options) - expr = env.compile( - "cel.expr.conformance.proto2.TestAllTypes{single_string: 'hello'}" + called = [] + + def show(x): + called.append(x) + return "ok" + + def show_pair(a, b): + called.append((a, b)) + return "ok" + + ext = cel.CelExtension( + "show_ext", + functions=[ + cel.FunctionDecl( + "show", + [ + cel.Overload( + "show_any", cel.Type.STRING, [cel.Type.DYN], impl=show + ), + cel.Overload( + "show_pair", + cel.Type.STRING, + [cel.Type.DYN, cel.Type.DYN], + impl=show_pair, + ), + ], + ) + ], + ) + env = cel.NewEnv( + extensions=[ext_optional.ExtOptional(), ext], + options=self.options, + container="cel.expr.conformance.proto2", ) - res = expr.eval(env.Activation()) - with self.assertRaisesRegex( - TypeError, "Couldn't find message class for type" - ): - res.plain_value() - del res - del expr - del env + res = env.compile(expr).eval(env.Activation()) + if expect_exception: + with self.assertRaisesRegex( + TypeError, "Couldn't find message class for type" + ): + res.plain_value() + else: + # During evaluation of the function argument, the proto type will + # fail to resolve to a message class, and the error will be + # wrapped in an ERROR type. The function itself will not be called. + self.assertEqual(res.type(), cel.Type.ERROR) + self.assertIn("Couldn't find message class for type", res.value()) + self.assertEqual(called, []) finally: message_factory.GetMessageClass = orig_get_message_class + def testCustomFunction_raisesException(self): + def fail(x): + raise ValueError(f"bad value: {x}") + + ext = cel.CelExtension( + "fail_ext", + functions=[ + cel.FunctionDecl( + "fail", + [ + cel.Overload( + "fail_int", cel.Type.STRING, [cel.Type.INT], impl=fail + ) + ], + ) + ], + ) + env = cel.NewEnv(extensions=[ext], options=self.options) + res = env.compile("fail(42)").eval(env.Activation()) + self.assertEqual(res.type(), cel.Type.ERROR) + self.assertIn("bad value: 42", res.value()) + + def testCustomFunction_invalidReturnWithBrokenRepr(self): + class BrokenReprCallable: + + def __call__(self, x): + del x + return "not_an_int" + + def __repr__(self): + raise RuntimeError("repr failed") + + env = cel.NewEnv( + functions=[ + cel.FunctionDecl( + "broken_repr", + [ + cel.Overload( + "broken_repr_int", + cel.Type.INT, + [cel.Type.INT], + impl=BrokenReprCallable(), + ) + ], + ) + ], + options=self.options, + ) + res = env.compile("broken_repr(1)").eval() + self.assertEqual(res.type(), cel.Type.ERROR) + self.assertIn( + "Unexpected value type for 'Python function '''", res.value() + ) + def testEvalList(self): res: cel.Value = self._eval( "[1, 'CEL', true]", expected_return_type=cel.Type.LIST @@ -741,6 +862,17 @@ def testDynType(self): {"value": "foo", "type": cel.Type.STRING}, {"value": b"foo", "type": cel.Type.BYTES}, {"value": bytearray("bar", "ascii"), "type": cel.Type.BYTES}, + { + "value": datetime.datetime( + 2025, 1, 1, tzinfo=datetime.timezone.utc + ), + "type": cel.Type.TIMESTAMP, + }, + { + "value": datetime.timedelta(seconds=15), + "type": cel.Type.DURATION, + }, + {"value": cel.Type.INT, "type": cel.Type.TYPE}, { "value": msg, "type": cel.Type("cel.expr.conformance.proto2.TestAllTypes"), @@ -891,6 +1023,69 @@ def testTypeInitSignature(self): cel.Type("cel.expr.conformance.proto2.TestAllTypes"), cel.Type("cel.expr.conformance.proto2.TestAllTypes"), ) + self.assertTrue(cel.Type.DYN.is_assignable_from(cel.Type.INT)) + self.assertTrue( + cel.Type.LIST.is_assignable_from(cel.Type.List(cel.Type.INT)) + ) + self.assertFalse(cel.Type.INT.is_assignable_from(cel.Type.DYN)) + self.assertTrue(cel.Type("~T").is_assignable_from(cel.Type("~T"))) + self.assertFalse(cel.Type("~T").is_assignable_from(cel.Type("~U"))) + self.assertFalse( + cel.Type.TYPE.is_assignable_from( + cel.Type.AbstractType("optional_type", [cel.Type.INT]) + ) + ) + self.assertTrue( + cel.Type.AbstractType( + "optional_type", [cel.Type.INT] + ).is_assignable_from( + cel.Type.AbstractType("optional_type", [cel.Type.INT]) + ) + ) + self.assertFalse( + cel.Type.AbstractType( + "optional_type", [cel.Type.INT] + ).is_assignable_from( + cel.Type.AbstractType("optional_type", [cel.Type.STRING]) + ) + ) + self.assertFalse( + cel.Type.AbstractType( + "optional_type", [cel.Type.INT] + ).is_assignable_from( + cel.Type.AbstractType("other_type", [cel.Type.INT]) + ) + ) + + box_int = cel.Type.AbstractType("my_box", [cel.Type.INT]) + box_dyn = cel.Type.AbstractType("my_box", [cel.Type.DYN]) + other_box_int = cel.Type.AbstractType("other_box", [cel.Type.INT]) + self.assertEqual(cel.Type("my_box"), box_int) + self.assertEqual(box_int.name(), "my_box") + self.assertTrue(box_dyn.is_assignable_from(box_int)) + self.assertTrue(box_int.is_assignable_from(box_int)) + self.assertFalse(other_box_int.is_assignable_from(box_int)) + self.assertTrue(cel.Type.AbstractType("my_box").is_assignable_from(box_int)) + self.assertFalse( + box_int.is_assignable_from(cel.Type.AbstractType("my_box")) + ) + self.assertFalse( + box_int.is_assignable_from( + cel.Type.AbstractType("my_box", [cel.Type.INT, cel.Type.STRING]) + ) + ) + self.assertEqual(cel.Type("google.protobuf.Timestamp"), cel.Type.TIMESTAMP) + self.assertEqual(cel.Type("google.protobuf.Duration"), cel.Type.DURATION) + + def testCheckedExprTypeParamReturnType(self): + checked_expr = checked_pb.CheckedExpr() + checked_expr.expr.id = 1 + checked_expr.expr.ident_expr.name = "var_tp" + checked_expr.type_map[1].type_param = "T" + packed = any_pb.Any() + packed.Pack(checked_expr) + expr: cel.Expression = self.env.deserialize(packed.SerializeToString()) + self.assertEqual(expr.return_type(), cel.Type("~T")) def testCelExpressionPersistence_checkedExpr(self): expr: cel.Expression = self.env.compile("var_msg.single_string") @@ -1064,6 +1259,19 @@ def testProtoMessageToCelValueError(self): self.assertEqual(res.type(), cel.Type.ERROR) self.assertIn("Custom pool error", str(res.value())) + def testTypeToCelValueError(self): + bad_env = cel.NewEnv( + _RaisingDescriptorPool(), + variables={"var_type": cel.Type.TYPE}, + options=self.options, + ) + expr = bad_env.compile("var_type", disable_check=True) + res = expr.eval( + data={"var_type": cel.Type("cel.expr.conformance.proto2.TestSomeTypes")} + ) + self.assertEqual(res.type(), cel.Type.ERROR) + self.assertIn("Custom pool error", str(res.value())) + def testValueOutlivesExpression(self): expr: cel.Expression = self.env.compile("'hello ' + 'world'") val: cel.Value = expr.eval() @@ -1080,6 +1288,464 @@ def testValueOutlivesExpression(self): del expr self.assertEqual(val.plain_value(), ["a", "b", "c"]) + def testOptionalPythonObject(self): + none_opt = cel.Optional.none() + self.assertEqual(none_opt, cel.Optional()) + self.assertFalse(none_opt.has_value()) + self.assertFalse(bool(none_opt)) + self.assertEqual(none_opt.or_value("fallback"), "fallback") + self.assertEqual(repr(none_opt), "optional.none()") + with self.assertRaisesRegex(ValueError, r"optional\.none\(\) dereference"): + none_opt.value() + + int_opt = cel.Optional.of(42) + self.assertTrue(int_opt.has_value()) + self.assertTrue(bool(int_opt)) + self.assertEqual(int_opt.value(), 42) + self.assertEqual(int_opt.or_value(0), 42) + self.assertEqual(repr(int_opt), "optional.of(42)") + self.assertEqual(int_opt, cel.Optional.of(42)) + self.assertNotEqual(int_opt, cel.Optional.of(43)) + self.assertNotEqual(int_opt, none_opt) + self.assertNotEqual(int_opt, 42) + self.assertNotEqual(none_opt, None) # pylint: disable=g-generic-assert + + false_opt = cel.Optional.of(False) + self.assertTrue(false_opt.has_value()) + self.assertTrue(bool(false_opt)) + self.assertIs(false_opt.value(), False) + + null_opt = cel.Optional.of(None) + self.assertTrue(null_opt.has_value()) + self.assertTrue(bool(null_opt)) + self.assertIsNone(null_opt.value()) + self.assertIsNone(null_opt.or_value("fallback")) + self.assertEqual(null_opt, cel.Optional.of(None)) + self.assertNotEqual(null_opt, none_opt) + + self.assertEqual(cel.Optional[int].__origin__, cel.Optional) + + nan = float("nan") + nan_opt = cel.Optional.of(nan) + self.assertNotEqual(nan_opt, nan_opt) + self.assertNotEqual(nan_opt, cel.Optional.of(float("nan"))) + + class RaisingEq: + + def __eq__(self, other: object) -> bool: + raise RuntimeError("boom") + + with self.assertRaisesRegex(RuntimeError, "boom"): + _ = cel.Optional.of(RaisingEq()) == cel.Optional.of(RaisingEq()) + + def testOptionalType(self): + self.assertEqual(cel.Type.OPTIONAL, cel.Type.Optional()) + self.assertEqual(cel.Type.OPTIONAL, cel.Type.Optional(cel.Type.DYN)) + self.assertEqual(cel.Type.OPTIONAL, cel.Type("optional_type")) + self.assertEqual(cel.Type.OPTIONAL, cel.Type("optional_type")) + self.assertEqual( + cel.Type.Optional(cel.Type.INT), cel.Type("optional_type") + ) + self.assertEqual( + cel.Type.Optional(cel.Type.INT), + cel.Type.AbstractType("optional_type", [cel.Type.INT]), + ) + self.assertTrue( + cel.Type.OPTIONAL.is_assignable_from(cel.Type.Optional(cel.Type.INT)) + ) + self.assertTrue( + cel.Type.Optional(cel.Type.INT).is_assignable_from( + cel.Type.Optional(cel.Type.INT) + ) + ) + self.assertFalse( + cel.Type.Optional(cel.Type.INT).is_assignable_from( + cel.Type.Optional(cel.Type.STRING) + ) + ) + self.assertFalse( + cel.Type.AbstractType("other_type", [cel.Type.INT]).is_assignable_from( + cel.Type.Optional(cel.Type.INT) + ) + ) + + def testEvalOptional(self): + res = self._eval( + "optional.of(42)", + expected_return_type=cel.Type.Optional(cel.Type.INT), + ) + self.assertEqual(res.type(), cel.Type.OPTIONAL) + self.assertEqual(res.plain_value(), cel.Optional.of(42)) + self.assertTrue(res.value().has_value()) + self.assertEqual(res.value().value().type(), cel.Type.INT) + self.assertEqual(res.value().value().value(), 42) + + res = self._eval("optional.none()", expected_return_type=cel.Type.OPTIONAL) + self.assertEqual(res.type(), cel.Type.OPTIONAL) + self.assertEqual(res.plain_value(), cel.Optional.none()) + self.assertEqual(res.value(), cel.Optional.none()) + + res = self._eval("optional.of(null)") + plain_opt = res.plain_value() + self.assertIsInstance(plain_opt, cel.Optional) + self.assertEqual(plain_opt, cel.Optional.of(None)) + self.assertIsNone(plain_opt.value()) + + res = self._eval("[optional.of(1), optional.none()]") + self.assertEqual( + res.plain_value(), [cel.Optional.of(1), cel.Optional.none()] + ) + + res = self._eval( + "{'present': optional.of('hi'), 'absent': optional.none()}" + ) + self.assertEqual( + res.plain_value(), + {"present": cel.Optional.of("hi"), "absent": cel.Optional.none()}, + ) + + res = self._eval( + "var_opt_int", + {"var_opt_int": cel.Optional.of(7)}, + expected_return_type=cel.Type.Optional(cel.Type.INT), + ) + self.assertEqual(res.plain_value(), cel.Optional.of(7)) + + res = self._eval( + "var_opt_int.orValue(99)", {"var_opt_int": cel.Optional.none()} + ) + self.assertEqual(res.value(), 99) + + res = self._eval("var_dyn", {"var_dyn": cel.Optional.of("abc")}) + self.assertEqual(res.type(), cel.Type.OPTIONAL) + self.assertEqual(res.plain_value(), cel.Optional.of("abc")) + + res = self._eval("var_opt_int", {"var_opt_int": 42}) + self.assertEqual(res.type(), cel.Type.ERROR) + self.assertIn( + "Unexpected value type for 'var_opt_int': int. (Expected" + " optional_type)", + res.value(), + ) + + res = self._eval("var_opt_int", {"var_opt_int": cel.Optional.of("bad")}) + self.assertEqual(res.type(), cel.Type.ERROR) + self.assertIn( + "Unexpected value type for 'var_opt_int.value()': str. (Expected INT)", + res.value(), + ) + + opaque_env = cel.NewEnv( + variables={ + "var_untyped_opt": cel.Type.AbstractType("optional_type"), + "var_opaque": cel.Type.AbstractType("other_type", [cel.Type.INT]), + }, + extensions=[ext_optional.ExtOptional()], + options=self.options, + ) + untyped_expr = opaque_env.compile("var_untyped_opt") + res = untyped_expr.eval( + opaque_env.Activation({"var_untyped_opt": cel.Optional.of(123)}) + ) + self.assertEqual(res.plain_value(), cel.Optional.of(123)) + + opaque_expr = opaque_env.compile("var_opaque") + res = opaque_expr.eval(opaque_env.Activation({"var_opaque": 123})) + self.assertEqual(res.type(), cel.Type.ERROR) + self.assertIn("Unimplemented opaque type: other_type", res.value()) + + def testOptionalEquals(self): + opt_42 = self._eval("optional.of(42)") + opt_43 = self._eval("optional.of(43)") + opt_none = self._eval("optional.none()") + opt_null = self._eval("optional.of(null)") + + # Basic optional equality with == and equals(). + self.assertEqual(opt_42, self._eval("optional.of(42)")) + self.assertEqual(opt_42, cel.Optional.of(42)) + self.assertEqual(cel.Optional.of(42), opt_42) + self.assertNotEqual(opt_42, opt_43) + self.assertNotEqual(opt_42, opt_none) + self.assertNotEqual(opt_42, self._eval("42")) + self.assertNotEqual(self._eval("42"), opt_42) + self.assertNotEqual(opt_42, 42) + self.assertNotEqual(42, opt_42) + self.assertTrue(opt_42.equals(self._eval("optional.of(42)"))) + self.assertFalse(opt_42.equals(opt_43)) + self.assertFalse(opt_42.equals(opt_none)) + self.assertFalse(opt_42.equals(self._eval("42"))) + self.assertFalse(self._eval("42").equals(opt_42)) + with self.assertRaises(TypeError): + non_cel_opt: Any = cel.Optional.of(42) + opt_42.equals(non_cel_opt) + + # Empty and null-holding optionals. + self.assertEqual(opt_none, self._eval("optional.none()")) + self.assertEqual(opt_none, cel.Optional.none()) + self.assertEqual(cel.Optional.none(), opt_none) + self.assertNotEqual(opt_none, opt_null) + self.assertNotEqual(opt_none, self._eval("null")) + self.assertNotEqual(self._eval("null"), opt_none) + self.assertNotEqual(opt_none, None) + self.assertNotEqual(None, opt_none) + self.assertTrue(opt_none.equals(self._eval("optional.none()"))) + self.assertFalse(opt_none.equals(opt_null)) + self.assertFalse(opt_none.equals(self._eval("null"))) + self.assertFalse(self._eval("null").equals(opt_none)) + + self.assertEqual(opt_null, self._eval("optional.of(null)")) + self.assertEqual(opt_null, cel.Optional.of(None)) + self.assertEqual(cel.Optional.of(None), opt_null) + self.assertTrue(opt_null.equals(self._eval("optional.of(null)"))) + self.assertFalse(opt_null.equals(opt_none)) + self.assertFalse(opt_null.equals(self._eval("null"))) + + # Cross-numeric, bool-vs-int, and precision differences between == and + # equals(). + opt_int = self._eval("optional.of(1)") + opt_uint = self._eval("optional.of(1u)") + opt_double = self._eval("optional.of(1.0)") + opt_bool = self._eval("optional.of(true)") + self.assertEqual(opt_int, opt_uint) + self.assertEqual(opt_int, opt_double) + self.assertEqual(opt_uint, opt_double) + self.assertTrue(opt_int.equals(opt_uint)) + self.assertTrue(opt_int.equals(opt_double)) + self.assertTrue(opt_uint.equals(opt_double)) + self.assertEqual(opt_bool, opt_int) + self.assertEqual(opt_bool, cel.Optional.of(1)) + self.assertFalse(opt_bool.equals(opt_int)) + + exact_int_opt = self._eval("optional.of(9007199254740993)") + approx_double_opt = self._eval("optional.of(9007199254740992.0)") + self.assertNotEqual(exact_int_opt, approx_double_opt) + self.assertTrue(exact_int_opt.equals(approx_double_opt)) + + # NaN inside optional is unequal to itself in both Python and CEL. + nan_opt = self._eval("optional.of(0.0 / 0.0)") + self.assertNotEqual(nan_opt, nan_opt) + self.assertNotEqual(nan_opt, self._eval("optional.of(0.0 / 0.0)")) + self.assertFalse(nan_opt.equals(nan_opt)) + self.assertFalse(nan_opt.equals(self._eval("optional.of(0.0 / 0.0)"))) + + # Nested optionals and containers with optionals. + self.assertEqual( + self._eval("optional.of(optional.of(1))"), + self._eval("optional.of(optional.of(1u))"), + ) + self.assertEqual( + self._eval("optional.of(optional.of(1))"), + cel.Optional.of(cel.Optional.of(1)), + ) + self.assertEqual( + cel.Optional.of(cel.Optional.of(1)), + self._eval("optional.of(optional.of(1))"), + ) + self.assertTrue( + self._eval("optional.of(optional.of(1))").equals( + self._eval("optional.of(optional.of(1u))") + ) + ) + self.assertEqual( + self._eval("optional.of(optional.of(true))"), + self._eval("optional.of(optional.of(1))"), + ) + self.assertFalse( + self._eval("optional.of(optional.of(true))").equals( + self._eval("optional.of(optional.of(1))") + ) + ) + self.assertEqual( + self._eval("optional.of(optional.none())"), + cel.Optional.of(cel.Optional.none()), + ) + self.assertTrue( + self._eval("optional.of(optional.none())").equals( + self._eval("optional.of(optional.none())") + ) + ) + + opt_list = self._eval("[optional.of(1), optional.none()]") + self.assertEqual(opt_list, [cel.Optional.of(1), cel.Optional.none()]) + self.assertEqual([cel.Optional.of(1), cel.Optional.none()], opt_list) + self.assertTrue( + opt_list.equals(self._eval("[optional.of(1u), optional.none()]")) + ) + self.assertFalse( + self._eval("[optional.of(true)]").equals(self._eval("[optional.of(1)]")) + ) + self.assertEqual(opt_list.value()[0], cel.Optional.of(1)) + self.assertTrue(opt_list.value()[0].equals(self._eval("optional.of(1u)"))) + + opt_map = self._eval("{'a': optional.of(1), 'b': optional.none()}") + self.assertEqual( + opt_map, {"a": cel.Optional.of(1), "b": cel.Optional.none()} + ) + self.assertTrue( + opt_map.equals( + self._eval("{'a': optional.of(1.0), 'b': optional.none()}") + ) + ) + self.assertEqual(opt_map.value()["a"], cel.Optional.of(1)) + self.assertTrue(opt_map.value()["a"].equals(self._eval("optional.of(1u)"))) + + self.assertTrue( + self._eval("optional.of([1, 2])").equals( + self._eval("optional.of([1u, 2.0])") + ) + ) + self.assertEqual(self._eval("optional.of([1, 2])"), cel.Optional.of([1, 2])) + + # Proto messages inside optionals. + msg1 = test_all_types_pb.TestAllTypes() + msg1.single_any.Pack( + test_all_types_pb.TestAllTypes(single_int32=1, single_string="a") + ) + msg2 = test_all_types_pb.TestAllTypes() + msg2.single_any.type_url = msg1.single_any.type_url + msg2.single_any.value = ( + test_all_types_pb.TestAllTypes(single_string="a").SerializeToString() + + test_all_types_pb.TestAllTypes(single_int32=1).SerializeToString() + ) + opt_msg1 = self._eval("optional.of(var_msg)", {"var_msg": msg1}) + opt_msg2 = self._eval("optional.of(var_msg)", {"var_msg": msg2}) + self.assertEqual(opt_msg1, cel.Optional.of(msg1)) + # == delegates to Python protobuf Message.__eq__ (whose Any handling varies + # across protobuf runtimes), whereas equals() uses CEL MessageEquals. + self.assertEqual(opt_msg1 == opt_msg2, msg1 == msg2) + self.assertTrue(opt_msg1.equals(opt_msg2)) + other_msg = test_all_types_pb.TestAllTypes(single_int32=2) + opt_other_msg = self._eval("optional.of(var_msg)", {"var_msg": other_msg}) + self.assertNotEqual(opt_msg1, opt_other_msg) + self.assertFalse(opt_msg1.equals(opt_other_msg)) + + # Error values vs optionals. + err_val = self._eval("var_bool", {}) + self.assertNotEqual(opt_42, err_val) + self.assertNotEqual(err_val, opt_42) + self.assertNotEqual(opt_none, err_val) + self.assertNotEqual(err_val, opt_none) + self.assertFalse(opt_42.equals(err_val)) + self.assertFalse(err_val.equals(opt_42)) + self.assertFalse(opt_none.equals(err_val)) + self.assertFalse(err_val.equals(opt_none)) + self.assertEqual(cel.Optional.of(err_val), cel.Optional.of(err_val)) + self.assertNotEqual( + cel.Optional.of(err_val), cel.Optional.of(err_val.value()) + ) + + # Custom matchers via reflected __eq__. + class _AlwaysEqual: + + def __eq__(self, other: object) -> bool: + return True + + always_equal = _AlwaysEqual() + self.assertEqual(opt_42, always_equal) + self.assertEqual(always_equal, opt_42) + self.assertEqual(opt_42, cel.Optional.of(always_equal)) + self.assertEqual(cel.Optional.of(always_equal), opt_42) + + def testOptionalWithPythonFunctions(self): + def show(x: Any) -> str: + return repr(x) + + def positive_only(n: int) -> cel.Optional[int]: + return cel.Optional.of(n) if n > 0 else cel.Optional.none() + + def unwrap_or_zero(opt: cel.Optional[int]) -> int: + return opt.or_value(0) + + env = cel.NewEnv( + variables={"m": cel.Type.DYN}, + extensions=[ext_optional.ExtOptional()], + functions=[ + cel.FunctionDecl( + "show", + [ + cel.Overload( + "show_any", + cel.Type.STRING, + [cel.Type("any")], + impl=show, + ) + ], + ), + cel.FunctionDecl( + "show_dyn", + [ + cel.Overload( + "show_dyn_ovl", + cel.Type.STRING, + [cel.Type.DYN], + impl=show, + ) + ], + ), + cel.FunctionDecl( + "positive_only", + [ + cel.Overload( + signature="positive_only(int)", + return_type=cel.Type.Optional(cel.Type.INT), + impl=positive_only, + ) + ], + ), + cel.FunctionDecl( + "unwrap_or_zero", + [ + cel.Overload( + "unwrap_or_zero_opt_int", + return_type=cel.Type.INT, + parameters=[cel.Type.Optional(cel.Type.INT)], + impl=unwrap_or_zero, + ) + ], + ), + ], + options=self.options, + ) + act = env.Activation({"m": {"a": 1}}) + + self.assertEqual( + env.compile("show(m[?'a'])").eval(act).value(), "optional.of(1)" + ) + self.assertEqual( + env.compile("show(m[?'missing'])").eval(act).value(), "optional.none()" + ) + self.assertEqual( + env.compile("show(optional.of(1))").eval(act).value(), "optional.of(1)" + ) + self.assertEqual( + env.compile("show({'k': optional.of(1)})").eval(act).value(), + "{'k': optional.of(1)}", + ) + self.assertEqual( + env.compile("show([optional.of(1), optional.none()])") + .eval(act) + .value(), + "[optional.of(1), optional.none()]", + ) + self.assertEqual( + env.compile("show_dyn(optional.of('ok'))").eval(act).value(), + "optional.of('ok')", + ) + self.assertEqual( + env.compile("positive_only(5).orValue(0)").eval(act).value(), 5 + ) + self.assertEqual( + env.compile("positive_only(-5).orValue(0)").eval(act).value(), 0 + ) + self.assertEqual( + env.compile("unwrap_or_zero(optional.of(5))").eval(act).value(), 5 + ) + self.assertEqual( + env.compile("unwrap_or_zero(optional.none())").eval(act).value(), 0 + ) + del act + del env + class CompatibleNumber: diff --git a/cel_expr_python/leak_check.py b/cel_expr_python/leak_check.py new file mode 100644 index 0000000..b8c4ea5 --- /dev/null +++ b/cel_expr_python/leak_check.py @@ -0,0 +1,137 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Reference and memory leak detection helpers for CEL Python tests.""" + +import array +import collections.abc +import functools +import gc +import sys +import tracemalloc +from typing import Any, TypeVar + +_DEFAULT_WARMUPS = 2 +_DEFAULT_RUNS = 3 + +_F = TypeVar("_F", bound=collections.abc.Callable[..., Any]) +_T = TypeVar("_T", bound=type) + + +def _live_blocks() -> int: + """Returns the allocated Python memory block count after clearing caches.""" + # Clear CPython's internal caches (e.g., the type attribute lookup cache) + # before sampling so one-time cached entries are not mistaken for leaks. + clear_caches = getattr(sys, "_clear_internal_caches", None) or getattr( + sys, "_clear_type_cache", None + ) + if clear_caches is not None: + clear_caches() + gc.collect() + return sys.getallocatedblocks() + + +def _tracemalloc_report( + run_once: collections.abc.Callable[[], None], + iterations: int = _DEFAULT_RUNS, + limit: int = 5, +) -> str: + """Re-runs `run_once` under tracemalloc and formats top growing sites.""" + was_tracing = tracemalloc.is_tracing() + if not was_tracing: + tracemalloc.start(10) + try: + run_once() + _live_blocks() + before = tracemalloc.take_snapshot() + for _ in range(iterations): + run_once() + _live_blocks() + after = tracemalloc.take_snapshot() + finally: + if not was_tracing: + tracemalloc.stop() + ignore = [tracemalloc.Filter(False, "")] + if tracemalloc.__file__: + ignore.append(tracemalloc.Filter(False, tracemalloc.__file__)) + stats = after.filter_traces(ignore).compare_to( + before.filter_traces(ignore), "traceback" + ) + lines = [] + sites = 0 + for stat in stats: + if stat.count_diff <= 0: + continue + lines.append( + f" {stat.count_diff:+d} blocks ({stat.size_diff:+d} B) allocated at:" + ) + lines.extend( + " " + line for line in stat.traceback.format(most_recent_first=True) + ) + sites += 1 + if sites >= limit: + break + return "\n".join(lines) or " (no growth attributed by tracemalloc)" + + +def assert_no_leaks( + test_method: _F, + *, + warmups: int = _DEFAULT_WARMUPS, + runs: int = _DEFAULT_RUNS, +) -> _F: + """Wraps a `unittest.TestCase` method to fail if live blocks grow every run. + + Uses CPython's `regrtest -R` strategy: runs `warmups` unmeasured iterations + to populate lazy process-wide caches, then `runs` measured iterations, + failing only if `sys.getallocatedblocks()` increases on every measured run. + + Args: + test_method: The test method to wrap. + warmups: Number of warm-up repetitions before measuring. + runs: Number of measured repetitions. + + Returns: + The wrapped test method. + """ + + @functools.wraps(test_method) + def wrapper(self: Any, *args: Any, **kwargs: Any) -> Any: + total_runs = warmups + runs + # Store samples in a C array so recording them keeps no Python ints alive. + blocks = array.array("q", [0]) * (total_runs + 1) + blocks[0] = _live_blocks() + for i in range(total_runs): + test_method(self, *args, **kwargs) + blocks[i + 1] = _live_blocks() + deltas = [blocks[i + 1] - blocks[i] for i in range(warmups, total_runs)] + if all(d > 0 for d in deltas): + self.fail( + f"Memory blocks grew on every one of {runs} repetitions" + f" (after {warmups} warm-up runs): deltas={deltas}." + " Top growing allocation sites:\n" + + _tracemalloc_report( + lambda: test_method(self, *args, **kwargs), iterations=runs + ) + ) + + return wrapper # type: ignore[return-value] + + +def leak_checked(cls: _T) -> _T: + """Class decorator wrapping every `test*` method with `assert_no_leaks`.""" + for name, attr in list(cls.__dict__.items()): + if name.startswith("test") and callable(attr): + setattr(cls, name, assert_no_leaks(attr)) + return cls diff --git a/cel_expr_python/policy/BUILD b/cel_expr_python/policy/BUILD new file mode 100644 index 0000000..e192f29 --- /dev/null +++ b/cel_expr_python/policy/BUILD @@ -0,0 +1,88 @@ +load("@pybind11_bazel//:build_defs.bzl", "pybind_extension") +load("@rules_python//python:py_test.bzl", "py_test") + +package(default_visibility = ["//visibility:public"]) + +pybind_extension( + name = "cel_policy", + srcs = [ + "py_cel_policy.cc", + "py_cel_policy.h", + "py_cel_policy_module.cc", + "py_cel_policy_yaml_parser.cc", + "py_cel_policy_yaml_parser.h", + ], + data = [ + "cel_policy.pyi", + "//cel_expr_python:cel", + ], + deps = [ + "//cel_expr_python:cel_compiler_capsule", + "//cel_expr_python:cel_extension", + "//cel_expr_python:status_macros", + "@com_google_absl//absl/status", + "@com_google_absl//absl/status:statusor", + "@com_google_absl//absl/strings", + "@com_google_absl//absl/types:span", + "@com_google_cel_cpp//common:ast_proto", + "@com_google_cel_cpp//common:source", + "@com_google_cel_cpp//compiler", + "@com_google_cel_cpp//policy:cel_policy", + "@com_google_cel_cpp//policy:cel_policy_parser", + "@com_google_cel_cpp//policy:cel_policy_validation_result", + "@com_google_cel_cpp//policy:compiler", + "@com_google_cel_cpp//policy:yaml_policy_parser", + "@com_google_cel_spec//proto/cel/expr:checked_cc_proto", + "@com_google_protobuf//:protobuf", + "@yaml-cpp", + ], +) + +py_test( + name = "cel_policy_yaml_test", + srcs = ["cel_policy_yaml_test.py"], + data = [ + "testdata/cel_policy.yaml", + "testdata/cel_policy_invalid.yaml", + ":cel_policy", + "//cel_expr_python:cel", + "//cel_expr_python/ext:ext_bindings", + "//cel_expr_python/ext:ext_optional", + ], + main = "cel_policy_yaml_test.py", + deps = [ + "//testing:proto3_test_all_types_py_pb2", + "@com_google_absl_py//absl/testing:absltest", + "@rules_python//python/runfiles", + ] + select({ + "@platforms//os:windows": [], + "//conditions:default": [ + ":cel_policy", + "//cel_expr_python:cel", + "//cel_expr_python/ext:ext_bindings", + "//cel_expr_python/ext:ext_optional", + ], + }), +) + +py_test( + name = "custom_policy_parser_test", + srcs = ["custom_policy_parser_test.py"], + data = [ + "testdata/custom_policy_format.yaml", + "testdata/custom_policy_format_with_errors.yaml", + ":cel_policy", + "//cel_expr_python:cel", + ], + main = "custom_policy_parser_test.py", + deps = [ + "@com_google_absl_py//absl/testing:absltest", + "@rules_python//python/runfiles", + ] + select({ + "@platforms//os:windows": [], + "//conditions:default": [ + ":cel_policy", + "//cel_expr_python:cel", + ], + }), +) diff --git a/cel_expr_python/policy/cel_policy.pyi b/cel_expr_python/policy/cel_policy.pyi new file mode 100644 index 0000000..4783708 --- /dev/null +++ b/cel_expr_python/policy/cel_policy.pyi @@ -0,0 +1,156 @@ +from collections.abc import MutableMapping, Sequence +from typing import Any, TypedDict + +from cel_expr_python import cel + +class ValueString: + @property + def id(self) -> int: ... + @property + def value(self) -> str: ... + def __repr__(self) -> str: ... + +class Import: + @property + def id(self) -> int: ... + @property + def name(self) -> ValueString: ... + def __repr__(self) -> str: ... + +class Variable: + @property + def name(self) -> ValueString: ... + @property + def expression(self) -> ValueString: ... + @property + def description(self) -> ValueString | None: ... + @property + def display_name(self) -> ValueString | None: ... + def __repr__(self) -> str: ... + +class OutputBlock: + def __init__(self, output: ValueString = ..., explanation: ValueString | None = ...) -> None: ... + @property + def output(self) -> ValueString: ... + @output.setter + def output(self, value: ValueString) -> None: ... + @property + def explanation(self) -> ValueString | None: ... + @explanation.setter + def explanation(self, value: ValueString | None) -> None: ... + def __repr__(self) -> str: ... + +class Match: + @property + def id(self) -> int: ... + @property + def condition(self) -> ValueString: ... + @property + def has_output_block(self) -> bool: ... + @property + def output_block(self) -> OutputBlock: ... + @property + def has_rule(self) -> bool: ... + @property + def rule(self) -> Rule: ... + def set_result(self, result: OutputBlock | Rule) -> None: ... + def __repr__(self) -> str: ... + +class Rule: + def __init__(self) -> None: ... + @property + def id(self) -> int: ... + @property + def rule_id(self) -> ValueString | None: ... + @property + def description(self) -> ValueString | None: ... + @property + def variables(self) -> Sequence[Variable]: ... + @variables.setter + def variables(self, value: Sequence[Variable]) -> None: ... + @property + def matches(self) -> list[Match]: ... + @matches.setter + def matches(self, value: Sequence[Match]) -> None: ... + @property + def is_aggregate(self) -> bool: ... + @is_aggregate.setter + def is_aggregate(self, value: bool) -> None: ... + def __repr__(self) -> str: ... + +class CelPolicy: + @property + def name(self) -> ValueString: ... + @property + def description(self) -> ValueString | None: ... + @property + def display_name(self) -> ValueString | None: ... + @property + def imports(self) -> Sequence[Import]: ... + @imports.setter + def imports(self, value: Sequence[Import]) -> None: ... + @property + def rule(self) -> Rule: ... + @rule.setter + def rule(self, value: Rule) -> None: ... + @property + def metadata(self) -> CelPolicyMetadata: ... + def __repr__(self) -> str: ... + +class CelPolicyMetadata(MutableMapping[str, Any]): ... + +class CelPolicySource: + def __repr__(self) -> str: ... + +class CelPolicyIssue: + def to_display_string(self, source: CelPolicySource) -> str: ... + +class CelPolicyParseResult: + @property + def is_valid(self) -> bool: ... + def policy(self) -> CelPolicy: ... + def release_policy(self) -> CelPolicy: ... + def issues(self) -> Sequence[CelPolicyIssue]: ... + def formatted_issues(self) -> str: ... + + + +class YAMLNode: + def is_scalar(self) -> bool: ... + def is_sequence(self) -> bool: ... + def is_map(self) -> bool: ... + def __len__(self) -> int: ... + def __getitem__(self, key: Any) -> Any: ... + +class CelPolicyParseContext: + @property + def policy(self) -> CelPolicy: ... + def report_error(self, id: int, message: str) -> None: ... + def collect_metadata(self, node: YAMLNode) -> int: ... + +class CelPolicyYamlParser: + def __init__(self) -> None: ... + def parse(self, yaml_content: str) -> CelPolicyParseResult: ... + def parse_policy_tag(self, ctx: CelPolicyParseContext, tag_name: ValueString, node: YAMLNode) -> bool: ... + def parse_rule(self, ctx: CelPolicyParseContext, node: YAMLNode, rule: Rule) -> None: ... + def parse_rule_tag(self, ctx: CelPolicyParseContext, tag_name: ValueString, node: YAMLNode, rule: Rule) -> bool: ... + def parse_variable(self, ctx: CelPolicyParseContext, node: YAMLNode, rule: Rule) -> Variable: ... + def parse_variable_tag(self, ctx: CelPolicyParseContext, tag_name: ValueString, node: YAMLNode, variable: Variable) -> bool: ... + def parse_match_tag(self, ctx: CelPolicyParseContext, tag_name: ValueString, node: YAMLNode, match: Match, rule: Rule) -> bool: ... + def parse_match(self, ctx: CelPolicyParseContext, node: YAMLNode, rule: Rule) -> Match: ... + def get_value_string(self, ctx: CelPolicyParseContext, node: YAMLNode, error_message: str) -> ValueString | None: ... + +def parse_yaml_cel_policy(yaml_content: str, parser: CelPolicyYamlParser | None = ..., description: str | None = ...) -> CelPolicyParseResult: ... + +class CompilePolicyOptions(TypedDict, total=False): + unnesting_height_limit: int + +class CelPolicyValidationResult: + @property + def is_valid(self) -> bool: ... + def expression(self) -> cel.Expression: ... + def issues(self) -> Sequence[CelPolicyIssue]: ... + def formatted_issues(self) -> str: ... + +def compile(env: cel.Env, policy: CelPolicy, options: CompilePolicyOptions | None = ...) -> CelPolicyValidationResult: ... + diff --git a/cel_expr_python/policy/cel_policy_yaml_test.py b/cel_expr_python/policy/cel_policy_yaml_test.py new file mode 100644 index 0000000..c87e13b --- /dev/null +++ b/cel_expr_python/policy/cel_policy_yaml_test.py @@ -0,0 +1,285 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from __future__ import annotations + +import gc + +from absl.testing import absltest +from python.runfiles import runfiles +from cel.expr.conformance.proto3 import test_all_types_pb2 +from cel_expr_python import cel +from cel_expr_python.ext import ext_bindings +from cel_expr_python.ext import ext_optional +from cel_expr_python.policy import cel_policy + +_TESTDATA_FILE_PATH = "_main/cel_expr_python/policy/testdata" + + +class PolicyTest(absltest.TestCase): + + def test_parse_yaml_cel_policy(self): + result = cel_policy.parse_yaml_cel_policy( + load_policy_yaml("cel_policy.yaml") + ) + self.assertTrue(result.is_valid) + + policy: cel_policy.CelPolicy = result.policy() + self.assertEqual(policy.name.value, "cel_policy") + + description: cel_policy.ValueString | None = policy.description + self.assertIsNotNone(description) + self.assertEqual(description.value, "A test policy for CEL") + + display_name: cel_policy.ValueString | None = policy.display_name + self.assertIsNotNone(display_name) + self.assertEqual(display_name.value, "Cel Policy") + + self.assertLen(policy.imports, 2) + self.assertEqual( + policy.imports[0].name.value, + "cel.expr.conformance.proto3.TestAllTypes", + ) + self.assertEqual( + policy.imports[1].name.value, + "cel.expr.conformance.proto3.TestAllTypes.NestedEnum", + ) + + rule: cel_policy.Rule = policy.rule + self.assertFalse(rule.is_aggregate) + + rule_id: cel_policy.ValueString | None = rule.rule_id + self.assertIsNotNone(rule_id) + self.assertEqual(rule_id.value, "test_rule") + + rule_description: cel_policy.ValueString | None = rule.description + self.assertIsNotNone(rule_description) + self.assertEqual(rule_description.value, "test rule description") + + self.assertLen(rule.variables, 1) + var: cel_policy.Variable = rule.variables[0] + self.assertEqual(var.name.value, "test_var") + + self.assertLen(rule.matches, 2) + match0: cel_policy.Match = rule.matches[0] + self.assertEqual( + match0.condition.value.strip(), + "spec.single_int32 > TestAllTypes{single_int64: 10}.single_int64", + ) + self.assertTrue(match0.has_output_block) + self.assertEqual( + match0.output_block.output.value.strip(), + '"invalid spec, got single_int32=" + string(spec.single_int32) + ' + '", wanted <= 10"', + ) + + def test_cel_policy_yaml_parser(self): + result = cel_policy.parse_yaml_cel_policy( + load_policy_yaml("cel_policy.yaml") + ) + policy = result.policy() + + self.assertEmpty(policy.metadata) + self.assertNotIn("key1", policy.metadata) + + policy.metadata["key1"] = "value1" + self.assertEqual(policy.metadata["key1"], "value1") + self.assertIn("key1", policy.metadata) + self.assertLen(policy.metadata, 1) + + self.assertEqual(list(policy.metadata), ["key1"]) + self.assertEqual(list(policy.metadata.keys()), ["key1"]) + self.assertEqual(list(policy.metadata.values()), ["value1"]) + self.assertEqual(list(policy.metadata.items()), [("key1", "value1")]) + + self.assertRegex(repr(policy), r"metadata:\s*\{\s*key1:\s*value1\s*}") + + def _compile_test_policy( + self, + filename: str, + options: cel_policy.CompilePolicyOptions | None = None, + ) -> cel_policy.CelPolicyValidationResult: + result = cel_policy.parse_yaml_cel_policy(load_policy_yaml(filename)) + policy = result.policy() + + env = cel.NewEnv( + variables={ + "spec": cel.Type("cel.expr.conformance.proto3.TestAllTypes") + }, + extensions=[ + ext_bindings.ExtBindings(), + ext_optional.ExtOptional(), + ], + ) + return cel_policy.compile(env, policy, options) + + def test_compile_policy_valid(self): + result = self._compile_test_policy( + "cel_policy.yaml", {"unnesting_height_limit": 10} + ) + self.assertTrue(result.is_valid, result.formatted_issues()) + self.assertIsNotNone(result.expression()) + + def test_compile_policy_invalid(self): + result = self._compile_test_policy("cel_policy_invalid.yaml") + self.assertFalse(result.is_valid) + self.assertIn( + "condition must evaluate to bool", + result.formatted_issues(), + ) + with self.assertRaises(RuntimeError): + result.expression() + + def test_compile_policy_invalid_env_type(self): + result = cel_policy.parse_yaml_cel_policy( + load_policy_yaml("cel_policy.yaml") + ) + policy = result.policy() + with self.assertRaises(TypeError): + cel_policy.compile(None, policy) # pyrefly: ignore[bad-argument-type] + with self.assertRaises(TypeError): + cel_policy.compile("not_an_env", policy) # pyrefly: ignore[bad-argument-type] + + def test_compile_policy_invalid_options_type(self): + with self.assertRaises(RuntimeError): + self._compile_test_policy( + "cel_policy.yaml", + {"unnesting_height_limit": "ten"}, # pyrefly: ignore[bad-argument-type] + ) + + def test_compile_policy_unnesting_height_limit_too_small(self): + with self.assertRaisesRegex( + RuntimeError, "unnesting_height_limit must be at least 2" + ): + self._compile_test_policy( + "cel_policy.yaml", {"unnesting_height_limit": 1} + ) + + def test_compile_policy_default_options(self): + result = self._compile_test_policy("cel_policy.yaml") + self.assertTrue(result.is_valid, result.formatted_issues()) + self.assertIsNotNone(result.expression()) + + def test_compile_policy_result_outlives_env(self): + # `_compile_test_policy` drops its only reference to the `cel.Env`. + result = self._compile_test_policy("cel_policy.yaml") + gc.collect() + self.assertTrue(result.is_valid, result.formatted_issues()) + expr = result.expression() + self.assertIs(expr, result.expression()) + self.assertEqual( + repr( + expr.eval( + data={"spec": test_all_types_pb2.TestAllTypes(single_int32=15)} + ) + ), + 'optional.of("invalid spec, got single_int32=15, wanted <= 10")', + ) + + def test_compile_policy_eval_match_first_condition(self): + result = self._compile_test_policy("cel_policy.yaml") + expr = result.expression() + + res = expr.eval( + data={"spec": test_all_types_pb2.TestAllTypes(single_int32=15)} + ) + self.assertEqual( + repr(res), + 'optional.of("invalid spec, got single_int32=15, wanted <= 10")', + ) + + def test_compile_policy_eval_match_second_condition(self): + result = self._compile_test_policy("cel_policy.yaml") + expr = result.expression() + + res = expr.eval( + data={ + "spec": test_all_types_pb2.TestAllTypes( + single_int32=5, + standalone_enum=test_all_types_pb2.TestAllTypes.NestedEnum.BAR, + ) + } + ) + self.assertEqual( + repr(res), + 'optional.of("invalid spec, reference to BAR is not allowed")', + ) + + def test_compile_policy_eval_fallthrough(self): + result = self._compile_test_policy("cel_policy.yaml") + expr = result.expression() + + spec = test_all_types_pb2.TestAllTypes( + single_int32=5, + standalone_enum=test_all_types_pb2.TestAllTypes.NestedEnum.FOO, + ) + res = expr.eval(data={"spec": spec}) + self.assertEqual(repr(res), "optional.none()") + + def test_rule_is_aggregate_property(self): + custom_rule = cel_policy.Rule() + self.assertFalse(custom_rule.is_aggregate) + custom_rule.is_aggregate = True + self.assertTrue(custom_rule.is_aggregate) + + def test_parse_and_compile_aggregate_policy(self): + yaml_content = """\ +name: aggregate_policy +rule: + aggregate: + - condition: "x > 0" + output: "'positive'" + - condition: "x % 2 == 0" + output: "'even'" +""" + result = cel_policy.parse_yaml_cel_policy(yaml_content) + self.assertTrue(result.is_valid, result.formatted_issues()) + policy = result.policy() + self.assertTrue(policy.rule.is_aggregate) + + env = cel.NewEnv( + variables={"x": cel.Type.INT}, + extensions=[ext_bindings.ExtBindings(), ext_optional.ExtOptional()], + ) + compiled = cel_policy.compile(env, policy) + self.assertTrue(compiled.is_valid, compiled.formatted_issues()) + expr = compiled.expression() + # Aggregate rules collect every matching output into a list. + self.assertEqual(expr.eval(data={"x": 4}).value(), ["positive", "even"]) + self.assertEqual(expr.eval(data={"x": -2}).value(), ["even"]) + self.assertEqual(expr.eval(data={"x": -1}).value(), []) + + # Mutating is_aggregate on policy.rule updates the policy in place and + # switches compilation back to first-match semantics (returning an optional + # of the first matching output instead of a list of all matching outputs). + policy.rule.is_aggregate = False + first_match_compiled = cel_policy.compile(env, policy) + self.assertTrue( + first_match_compiled.is_valid, first_match_compiled.formatted_issues() + ) + self.assertEqual( + repr(first_match_compiled.expression().eval(data={"x": 4})), + 'optional.of("positive")', + ) + + +def load_policy_yaml(filename: str) -> str: + r = runfiles.Create() + path = r.Rlocation(f"{_TESTDATA_FILE_PATH}/{filename}") + with open(path, "rt") as f: + return f.read() + + +if __name__ == "__main__": + absltest.main() diff --git a/cel_expr_python/policy/custom_policy_parser_test.py b/cel_expr_python/policy/custom_policy_parser_test.py new file mode 100644 index 0000000..2933708 --- /dev/null +++ b/cel_expr_python/policy/custom_policy_parser_test.py @@ -0,0 +1,333 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from absl.testing import absltest +from python.runfiles import runfiles +from cel_expr_python.policy import cel_policy + +_TESTDATA_FILE_PATH = "_main/cel_expr_python/policy/testdata" + + +class TestCustomYamlPolicyParser(cel_policy.CelPolicyYamlParser): + """TestCustomYamlPolicyParser is used in unit tests for custom policy formats. + + It demonstrates the versatility of the cel::YamlPolicyParser framework API in + Python by implementing custom tag and block parsing without needing to modify + the core parser. + """ + + def __init__(self): + super().__init__() + self.log = [] + + def parse_rule( + self, + ctx: cel_policy.CelPolicyParseContext, + node: cel_policy.YAMLNode, + rule: cel_policy.Rule, + ) -> None: + self.log.append("parse_rule") + return super().parse_rule(ctx, node, rule) + + def parse_rule_tag( + self, + ctx: cel_policy.CelPolicyParseContext, + tag_name: cel_policy.ValueString, + node: cel_policy.YAMLNode, + rule: cel_policy.Rule, + ) -> bool: + self.log.append(f"parse_rule_tag: {tag_name.value}") + return super().parse_rule_tag(ctx, tag_name, node, rule) + + def parse_policy_tag( + self, + ctx: cel_policy.CelPolicyParseContext, + tag_name: cel_policy.ValueString, + node: cel_policy.YAMLNode, + ) -> bool: + self.log.append(f"parse_policy_tag: {tag_name.value}") + if tag_name.value in ("name", "description", "imports"): + return super().parse_policy_tag(ctx, tag_name, node) + + if tag_name.value == "inputs": + if not node.is_sequence(): + ctx.report_error(tag_name.id, "Policy 'inputs' is not a sequence") + return True + + rule: cel_policy.Rule = ctx.policy.rule + vars_list: list[cel_policy.Variable] = [] + for _, var_node in enumerate(node): # pyrefly: ignore[bad-argument-type] + var: cel_policy.Variable = self.parse_variable(ctx, var_node, rule) + vars_list.append(var) + rule.variables = vars_list + + return True + + if tag_name.value == "purpose": + purpose: cel_policy.ValueString | None = self.get_value_string( + ctx, node, "Policy purpose is not a string" + ) + if purpose is not None: + ctx.policy.metadata["purpose"] = purpose.value + return True + + if tag_name.value == "version": + version: cel_policy.ValueString | None = self.get_value_string( + ctx, node, "Policy version is not a string" + ) + if version is None: + return True + try: + version_int: int = int(version.value) + except ValueError: + ctx.report_error( + version.id, f"Policy version is not an integer: {version.value}" + ) + return True + ctx.policy.metadata["version"] = version_int + return True + + if tag_name.value == "conditions": + if not node.is_sequence(): + ctx.report_error(tag_name.id, "Policy 'conditions' is not a sequence") + return True + + rule: cel_policy.Rule = ctx.policy.rule + for _, condition in enumerate(node): # pyrefly: ignore[bad-argument-type] + # Track the number of existing matches before parsing. When ParseMatch + # evaluates an 'else' block, it recursively triggers parsing and adds + # internal inner matches directly to the rule's match vector. + # Inserting the outer match at begin() + size_before ensures that the + # primary outer 'if' condition is always evaluated before its nested + # 'else' fallbacks. + size_before = len(rule.matches) + match = self.parse_match(ctx, condition, rule) + rule.matches.insert(size_before, match) + + return True + + return False + + def parse_match( + self, + ctx: cel_policy.CelPolicyParseContext, + node: cel_policy.YAMLNode, + rule: cel_policy.Rule, + ) -> cel_policy.Match: + self.log.append("parse_match") + return super().parse_match(ctx, node, rule) + + def parse_match_tag( + self, + ctx: cel_policy.CelPolicyParseContext, + tag_name: cel_policy.ValueString, + node: cel_policy.YAMLNode, + match: cel_policy.Match, + rule: cel_policy.Rule, + ) -> bool: + self.log.append(f"parse_match_tag: {tag_name.value}") + if tag_name.value == "if": + condition: cel_policy.ValueString | None = self.get_value_string( + ctx, node, "Policy 'if' condition is not a string" + ) + if condition is not None: + match.condition = condition # pyrefly: ignore[read-only] + return True + if tag_name.value == "then": + self.parse_then_block(ctx, node, match) + return True + if tag_name.value == "else": + self.parse_else_block(ctx, node, rule) + return True + return False + + def parse_then_block( + self, + ctx: cel_policy.CelPolicyParseContext, + value_node: cel_policy.YAMLNode, + match: cel_policy.Match, + ): + if value_node.is_scalar(): + val: cel_policy.ValueString | None = self.get_value_string( + ctx, value_node, "Policy condition 'then' is not a string" + ) + if val is not None: + output: cel_policy.OutputBlock = cel_policy.OutputBlock() + output.output = val + match.set_result(output) + elif value_node.is_map(): + nested_rule: cel_policy.Rule = cel_policy.Rule() + nested_match: cel_policy.Match = self.parse_match( + ctx, value_node, nested_rule + ) + nested_rule.matches.insert(0, nested_match) + match.set_result(nested_rule) + else: + id_val: int = ctx.collect_metadata(value_node) + ctx.report_error(id_val, "Bad syntax in 'if/then' block") + + def parse_else_block( + self, + ctx: cel_policy.CelPolicyParseContext, + value_node: cel_policy.YAMLNode, + rule: cel_policy.Rule, + ): + if value_node.is_scalar(): + val: cel_policy.ValueString | None = self.get_value_string( + ctx, value_node, "Policy condition 'else' is not a string" + ) + if val is not None: + else_match: cel_policy.Match = cel_policy.Match() + else_match.id = ctx.collect_metadata(value_node) # pyrefly: ignore[read-only] + output: cel_policy.OutputBlock = cel_policy.OutputBlock() + output.output = val + else_match.set_result(output) + rule.matches.append(else_match) + elif value_node.is_map(): + size_before: int = len(rule.matches) + match: cel_policy.Match = self.parse_match(ctx, value_node, rule) + rule.matches.insert(size_before, match) + else: + id_val: int = ctx.collect_metadata(value_node) + ctx.report_error(id_val, "Bad syntax in 'if/then' block") + + def parse_variable( + self, + ctx: cel_policy.CelPolicyParseContext, + node: cel_policy.YAMLNode, + rule: cel_policy.Rule, + ) -> cel_policy.Variable: + self.log.append("parse_variable") + return super().parse_variable(ctx, node, rule) + + def parse_variable_tag( + self, + ctx: cel_policy.CelPolicyParseContext, + tag_name: cel_policy.ValueString, + node: cel_policy.YAMLNode, + variable: cel_policy.Variable, + ) -> bool: + self.log.append(f"parse_variable_tag: {tag_name.value}") + if tag_name.value == "name": + return super().parse_variable_tag(ctx, tag_name, node, variable) + if tag_name.value == "assign": + expression: cel_policy.ValueString | None = self.get_value_string( + ctx, node, "Variable 'assign' expression is not a string" + ) + if expression is not None: + variable.expression = expression # pyrefly: ignore[read-only] + return True + return False + + +class CustomPolicyParserTest(absltest.TestCase): + + def test_custom_policy_format(self): + yaml_content: str = load_policy_yaml("custom_policy_format.yaml") + + parser: TestCustomYamlPolicyParser = TestCustomYamlPolicyParser() + result: cel_policy.CelPolicyParseResult = cel_policy.parse_yaml_cel_policy( + yaml_content, parser + ) + + self.assertTrue(result.is_valid, result.formatted_issues()) + + self.assertEqual( + parser.log, + [ + "parse_policy_tag: name", + "parse_policy_tag: description", + "parse_policy_tag: imports", + "parse_policy_tag: purpose", + "parse_policy_tag: version", + "parse_policy_tag: inputs", + "parse_variable", + "parse_variable_tag: name", + "parse_variable_tag: assign", + "parse_policy_tag: conditions", + "parse_match", + "parse_match_tag: if", + "parse_match_tag: then", + "parse_match_tag: else", + "parse_match", + "parse_match_tag: if", + "parse_match_tag: then", + "parse_match", + "parse_match_tag: if", + "parse_match_tag: then", + "parse_match_tag: else", + "parse_match_tag: else", + ], + ) + + policy = result.policy() + self.assertEqual(policy.name.value, "cel_policy_custom_tags") + description = policy.description + self.assertIsNotNone(description) + self.assertEqual(description.value, "A custom policy format") + self.assertEqual(policy.metadata["purpose"], "test") + self.assertEqual(policy.metadata["version"], 42) + self.assertLen(policy.imports, 1) + self.assertEqual( + policy.imports[0].name.value, + "cel.expr.conformance.proto3.TestAllTypes", + ) + self.assertLen(policy.rule.variables, 1) + self.assertEqual(policy.rule.variables[0].name.value, "test_input") + self.assertEqual(policy.rule.variables[0].expression.value, "42") + + def test_custom_policy_format_with_errors(self): + yaml_content: str = load_policy_yaml( + "custom_policy_format_with_errors.yaml" + ) + + result: cel_policy.CelPolicyParseResult = cel_policy.parse_yaml_cel_policy( + yaml_content, + TestCustomYamlPolicyParser(), + description="custom_policy_format_with_errors.yaml", + ) + + self.assertFalse(result.is_valid) + formatted_issues: str = result.formatted_issues() + self.assertIn( + "custom_policy_format_with_errors.yaml:" + "19:3: Policy purpose is not a string", + formatted_issues, + ) + self.assertIn( + "custom_policy_format_with_errors.yaml:" + "20:10: Policy version is not an integer: new", + formatted_issues, + ) + self.assertIn( + "custom_policy_format_with_errors.yaml:" + "23:5: Policy 'if' condition is not a string", + formatted_issues, + ) + self.assertIn( + "custom_policy_format_with_errors.yaml:" + "31:7: Bad syntax in 'if/then' block", + formatted_issues, + ) + + +def load_policy_yaml(filename: str) -> str: + r = runfiles.Create() + path = r.Rlocation(f"{_TESTDATA_FILE_PATH}/{filename}") + with open(path, "rt") as f: + return f.read() + + +if __name__ == "__main__": + absltest.main() diff --git a/cel_expr_python/policy/py_cel_policy.cc b/cel_expr_python/policy/py_cel_policy.cc new file mode 100644 index 0000000..b96b211 --- /dev/null +++ b/cel_expr_python/policy/py_cel_policy.cc @@ -0,0 +1,416 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "cel_expr_python/policy/py_cel_policy.h" + +#include +#include +#include +#include +#include +#include +#include +#include + +#include "cel/expr/checked.pb.h" +#include "google/protobuf/any.pb.h" +#include "absl/status/status.h" +#include "absl/status/statusor.h" +#include "absl/strings/str_cat.h" +#include "absl/strings/str_join.h" +#include "absl/strings/str_split.h" +#include "absl/strings/string_view.h" +#include "absl/types/span.h" +#include "common/ast_proto.h" +#include "compiler/compiler.h" +#include "policy/cel_policy.h" +#include "policy/cel_policy_parse_result.h" +#include "policy/cel_policy_validation_result.h" +#include "policy/compiler.h" +#include "cel_expr_python/cel_compiler_capsule.h" +#include "cel_expr_python/py_error_status.h" +#include "cel_expr_python/status_macros.h" +#include +#include + +namespace cel_python { + +namespace py = ::pybind11; + +namespace { + +std::string IndentBlock(absl::string_view text) { + if (text.empty()) { + return ""; + } + std::vector lines; + for (absl::string_view line : absl::StrSplit(text, '\n')) { + if (line.empty()) { + lines.push_back(""); + } else { + lines.push_back(absl::StrCat(" ", line)); + } + } + return absl::StrJoin(lines, "\n"); +} + +std::string MetadataValueDebugString(std::any value) { + if (value.type() == typeid(py::object)) { + return py::str(std::any_cast(value)).cast(); + } + if (value.type() == typeid(std::monostate)) { + return "null"; + } + if (value.type() == typeid(cel::ValueString)) { + return std::any_cast(value).DebugString(); + } + if (value.type() == typeid(bool)) { + return std::any_cast(value) ? "true" : "false"; + } + if (value.type() == typeid(int)) { + return absl::StrCat(std::any_cast(value)); + } + if (value.type() == typeid(std::string)) { + return std::any_cast(value); + } + return absl::StrCat("typeid: ", value.type().name()); +} + +// This function is identical to the C++ version in cel_policy.cc except that +// it uses __repr__ to get the string representation of the metadata values. +std::string CelPolicyDebugString(const cel::CelPolicy& policy) { + std::string result; + absl::StrAppend(&result, "CelPolicy{\n"); + absl::StrAppend( + &result, + " ===========================================================\n"); + if (policy.source() != nullptr) { + absl::StrAppend(&result, + IndentBlock(IndentBlock(policy.source()->DebugString())), + "\n"); + } + absl::StrAppend( + &result, + " ===========================================================\n"); + absl::StrAppend(&result, " name: ", policy.name().DebugString(), "\n"); + if (policy.description().has_value()) { + absl::StrAppend( + &result, " description: ", policy.description()->DebugString(), "\n"); + } + if (policy.display_name().has_value()) { + absl::StrAppend(&result, + " display_name: ", policy.display_name()->DebugString(), + "\n"); + } + if (!policy.metadata().empty()) { + std::vector sorted_keys; + for (const auto& [key, _] : policy.metadata()) { + sorted_keys.push_back(key); + } + std::sort(sorted_keys.begin(), sorted_keys.end()); + + absl::StrAppend(&result, " metadata: {\n"); + for (const auto& key : sorted_keys) { + const auto& value = policy.metadata().at(key); + absl::StrAppend(&result, " ", key, ": ", + MetadataValueDebugString(value), "\n"); + } + absl::StrAppend(&result, " }\n"); + } + if (!policy.imports().empty()) { + absl::StrAppend(&result, " imports:\n"); + for (const auto& import : policy.imports()) { + absl::StrAppend(&result, " ", import.DebugString(), "\n"); + } + } + absl::StrAppend(&result, IndentBlock(policy.rule().DebugString()), "\n"); + absl::StrAppend(&result, "}"); + return result; +} + +} // namespace + +class CelPolicyMetadataWrapper { + public: + // The lifetime of CelPolicyMetadataWrapper is tied to the lifetime of the + // CelPolicy object, see py::keep_alive in the CelPolicy bindings below. + cel::CelPolicy* policy; + + static void DefineBindings(pybind11::module& m); +}; + +class PyCelPolicyValidationResult { + public: + // `env` is the `cel.Env` the policy was compiled with. The compiled + // expression is built once here so that `expression()` is a cheap, + // thread-safe lookup. + PyCelPolicyValidationResult(cel::CelPolicyValidationResult result, + const py::object& env) + : result_(std::move(result)), + expression_(BuildExpression(result_, env)) {} + + bool IsValid() const { return result_.IsValid(); } + + absl::Span GetIssues() const { + return result_.GetIssues(); + } + + std::string FormattedIssues() const { return result_.FormatIssues(); } + + py::object GetExpression() const { return ThrowIfError(expression_); } + + private: + static absl::StatusOr BuildExpression( + const cel::CelPolicyValidationResult& result, const py::object& env) { + if (!result.IsValid() || result.GetAst() == nullptr) { + return absl::InvalidArgumentError(result.FormatIssues()); + } + cel::expr::CheckedExpr checked_expr; + CEL_PYTHON_RETURN_IF_ERROR( + cel::AstToCheckedExpr(*result.GetAst(), &checked_expr)); + // Hand the expression back to the `cel` module in serialized form: + // generated protobuf descriptors are per shared library, so a CheckedExpr + // message cannot be passed directly between `cel_policy` and `cel`. + google::protobuf::Any any; + if (!any.PackFrom(checked_expr)) { + return absl::InternalError("Failed to pack compiled policy expression."); + } + return env.attr("deserialize")(py::bytes(any.SerializeAsString())); + } + + cel::CelPolicyValidationResult result_; + // Must be declared after `result_`, which is used to initialize it. + absl::StatusOr expression_; +}; + +void DefinePolicyBindings(pybind11::module& m) { + CelPolicyMetadataWrapper::DefineBindings(m); + + py::class_(m, "ValueString") + .def(py::init<>()) + .def(py::init()) + .def_property_readonly("id", &cel::ValueString::id) + .def_property_readonly("value", &cel::ValueString::value) + .def("__repr__", &cel::ValueString::DebugString); + + py::class_(m, "Import") + .def_property_readonly("id", &cel::Import::id) + .def_property_readonly("name", &cel::Import::name) + .def("__repr__", &cel::Import::DebugString); + + py::class_(m, "Variable") + .def(py::init<>()) + .def_property("name", &cel::Variable::name, &cel::Variable::set_name) + .def_property("expression", &cel::Variable::expression, + &cel::Variable::set_expression) + .def_property("description", &cel::Variable::description, + &cel::Variable::set_description) + .def_property("display_name", &cel::Variable::display_name, + &cel::Variable::set_display_name) + .def("__repr__", &cel::Variable::DebugString); + + py::class_(m, "OutputBlock") + .def(py::init<>()) + .def(py::init>()) + .def_property("output", &cel::OutputBlock::output, + &cel::OutputBlock::set_output) + .def_property("explanation", &cel::OutputBlock::explanation, + &cel::OutputBlock::set_explanation) + .def("__repr__", &cel::OutputBlock::DebugString); + + py::class_(m, "Match") + .def(py::init<>()) + .def_property("id", &cel::Match::id, &cel::Match::set_id) + .def_property("condition", &cel::Match::condition, + &cel::Match::set_condition) + .def_property_readonly("has_output_block", &cel::Match::has_output_block) + .def_property_readonly( + "output_block", + [](const cel::Match& self) -> const cel::OutputBlock& { + return self.output_block(); + }, + py::return_value_policy::reference_internal) + .def_property_readonly("has_rule", &cel::Match::has_rule) + .def_property_readonly( + "rule", + [](const cel::Match& self) -> const cel::Rule& { + return self.rule(); + }, + py::return_value_policy::reference_internal) + .def("set_result", + py::overload_cast(&cel::Match::set_result)) + .def("set_result", + [](cel::Match& self, const cel::Rule& rule) { + self.set_result(std::make_unique(rule)); + }) + .def("__repr__", &cel::Match::DebugString); + + py::class_(m, "Rule") + .def(py::init<>()) + .def_property("id", &cel::Rule::id, &cel::Rule::set_id) + .def_property("rule_id", &cel::Rule::rule_id, &cel::Rule::set_rule_id) + .def_property("description", &cel::Rule::description, + &cel::Rule::set_description) + .def_property( + "variables", &cel::Rule::variables, + [](cel::Rule& self, const std::vector& vars) { + self.mutable_variables() = vars; + }) + .def_property( + "matches", &cel::Rule::matches, + [](cel::Rule& self, const std::vector& matches) { + self.mutable_matches() = matches; + }) + .def_property("is_aggregate", &cel::Rule::is_aggregate, + &cel::Rule::set_is_aggregate) + .def("__repr__", &cel::Rule::DebugString); + + py::class_(m, "CelPolicy") + .def_property("name", &cel::CelPolicy::name, &cel::CelPolicy::set_name) + .def_property("description", &cel::CelPolicy::description, + &cel::CelPolicy::set_description) + .def_property("display_name", &cel::CelPolicy::display_name, + &cel::CelPolicy::set_display_name) + .def_property( + "imports", &cel::CelPolicy::imports, + [](cel::CelPolicy& self, const std::vector& imports) { + self.mutable_imports() = imports; + }) + .def_property("rule", &cel::CelPolicy::rule, + [](cel::CelPolicy& self, const cel::Rule& rule) { + self.mutable_rule() = rule; + }) + .def_property_readonly("metadata", + py::cpp_function( + [](cel::CelPolicy& self) { + return CelPolicyMetadataWrapper{&self}; + }, + // Keep the CelPolicy alive as long as the + // CelPolicyMetadataWrapper is alive. 0 is + // self, 1 is the CelPolicyMetadataWrapper. + py::keep_alive<0, 1>())) + .def("__repr__", &CelPolicyDebugString); + + py::class_>( + m, "CelPolicySource") + .def("__repr__", &cel::CelPolicySource::DebugString); + + py::class_(m, "CelPolicyIssue") + .def("to_display_string", [](const cel::CelPolicyIssue& self, + const cel::CelPolicySource& src) { + // resolve ambiguous overload + return self.ToDisplayString(src); + }); + + py::class_(m, "CelPolicyParseResult") + .def_property_readonly("is_valid", &cel::CelPolicyParseResult::IsValid) + .def("policy", &cel::CelPolicyParseResult::GetPolicy, + py::return_value_policy::reference_internal) + .def("issues", &cel::CelPolicyParseResult::GetIssues) + .def("formatted_issues", &cel::CelPolicyParseResult::FormattedIssues); + + py::class_(m, "CelPolicyValidationResult") + .def_property_readonly("is_valid", &PyCelPolicyValidationResult::IsValid) + .def("expression", &PyCelPolicyValidationResult::GetExpression) + .def("issues", &PyCelPolicyValidationResult::GetIssues) + .def("formatted_issues", &PyCelPolicyValidationResult::FormattedIssues); + + m.def( + "compile", + [](py::object env, const cel::CelPolicy& policy, + std::optional options_dict) -> PyCelPolicyValidationResult { + std::shared_ptr compiler = GetCelCompiler(env); + cel::CompilePolicyOptions options; + if (options_dict) { + if (options_dict->contains("unnesting_height_limit")) { + options.unnesting_height_limit = + (*options_dict)["unnesting_height_limit"].cast(); + } + } + cel::CelPolicyValidationResult result = [&]() { + py::gil_scoped_release gil_release; + return ThrowIfError(cel::CompilePolicy(*compiler, policy, options)); + }(); + return PyCelPolicyValidationResult(std::move(result), env); + }, + py::arg("env"), py::arg("policy"), py::arg("options") = py::none()); +} + +void CelPolicyMetadataWrapper::DefineBindings(pybind11::module& m) { + py::class_(m, "CelPolicyMetadata") + .def("__getitem__", + [](CelPolicyMetadataWrapper& self, const std::string& key) { + auto it = self.policy->metadata().find(key); + if (it == self.policy->metadata().end()) { + throw py::key_error(key); + } + if (it->second.type() == typeid(py::object)) { + return std::any_cast(it->second); + } + return py::none().cast(); + }) + .def("__setitem__", + [](CelPolicyMetadataWrapper& self, const std::string& key, + py::object value) { + self.policy->mutable_metadata()[key] = value; + }) + .def("__contains__", + [](CelPolicyMetadataWrapper& self, const std::string& key) { + return self.policy->metadata().contains(key); + }) + .def("__len__", + [](CelPolicyMetadataWrapper& self) { + return self.policy->metadata().size(); + }) + .def( + "__iter__", + [](CelPolicyMetadataWrapper& self) { + return py::make_key_iterator(self.policy->metadata().begin(), + self.policy->metadata().end()); + }, + py::keep_alive<0, 1>()) + .def("keys", + [](CelPolicyMetadataWrapper& self) { + py::list l; + for (const auto& item : self.policy->metadata()) { + l.append(item.first); + } + return l; + }) + .def("values", + [](CelPolicyMetadataWrapper& self) { + py::list l; + for (const auto& item : self.policy->metadata()) { + if (item.second.type() == typeid(py::object)) { + l.append(std::any_cast(item.second)); + } else { + l.append(py::none()); + } + } + return l; + }) + .def("items", [](CelPolicyMetadataWrapper& self) { + py::list l; + for (const auto& item : self.policy->metadata()) { + py::object val = py::none(); + if (item.second.type() == typeid(py::object)) { + val = std::any_cast(item.second); + } + l.append(py::make_tuple(item.first, val)); + } + return l; + }); +} + +} // namespace cel_python diff --git a/cel_expr_python/policy/py_cel_policy.h b/cel_expr_python/policy/py_cel_policy.h new file mode 100644 index 0000000..56b8656 --- /dev/null +++ b/cel_expr_python/policy/py_cel_policy.h @@ -0,0 +1,26 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef THIRD_PARTY_CEL_PYTHON_POLICY_PY_CEL_POLICY_H_ +#define THIRD_PARTY_CEL_PYTHON_POLICY_PY_CEL_POLICY_H_ + +#include + +namespace cel_python { + +void DefinePolicyBindings(pybind11::module& m); + +} // namespace cel_python + +#endif // THIRD_PARTY_CEL_PYTHON_POLICY_PY_CEL_POLICY_H_ diff --git a/cel_expr_python/policy/py_cel_policy_module.cc b/cel_expr_python/policy/py_cel_policy_module.cc new file mode 100644 index 0000000..22f7c77 --- /dev/null +++ b/cel_expr_python/policy/py_cel_policy_module.cc @@ -0,0 +1,30 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "cel_expr_python/cel_extension.h" +#include "cel_expr_python/policy/py_cel_policy.h" +#include "cel_expr_python/policy/py_cel_policy_yaml_parser.h" +#include + +namespace cel_python { + +PYBIND11_MODULE(cel_policy, m, pybind11::mod_gil_not_used()) { + pybind11::module_::import(CEL_MODULE_NAME); + m.doc() = "Python bindings for CEL Policy."; + + DefinePolicyBindings(m); + DefineCelPolicyYamlParserBindings(m); +} + +} // namespace cel_python diff --git a/cel_expr_python/policy/py_cel_policy_yaml_parser.cc b/cel_expr_python/policy/py_cel_policy_yaml_parser.cc new file mode 100644 index 0000000..16c02bc --- /dev/null +++ b/cel_expr_python/policy/py_cel_policy_yaml_parser.cc @@ -0,0 +1,371 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "cel_expr_python/policy/py_cel_policy_yaml_parser.h" + +#include +#include +#include +#include +#include + +#include "absl/status/status.h" +#include "absl/status/statusor.h" +#include "absl/strings/string_view.h" +#include "common/source.h" +#include "policy/cel_policy.h" +#include "policy/cel_policy_parse_context.h" +#include "policy/cel_policy_parse_result.h" +#include "policy/yaml_policy_parser.h" +#include "cel_expr_python/py_error_status.h" +#include +#include +#include "yaml-cpp/node/node.h" +#include "yaml-cpp/node/parse.h" +#include "yaml-cpp/yaml.h" // IWYU pragma: keep + +namespace cel_python { + +namespace py = ::pybind11; + +class PyCelPolicyYamlParser : public cel::YamlPolicyParser { + public: + using cel::YamlPolicyParser::YamlPolicyParser; + + absl::StatusOr ParsePolicyTag(cel::CelPolicyParseContext& ctx, + const cel::ValueString& tag_name, + const YAML::Node& node) const override { + pybind11::gil_scoped_acquire gil; + pybind11::function override = + pybind11::get_override(this, "parse_policy_tag"); + if (override) { + auto py_result = override( + pybind11::cast(&ctx, pybind11::return_value_policy::reference), + tag_name, node); + return py_result.cast(); + } + pybind11::gil_scoped_release gil_release; + return cel::YamlPolicyParser::ParsePolicyTag(ctx, tag_name, node); + } + + absl::StatusOr SuperParsePolicyTag(cel::CelPolicyParseContext& ctx, + const cel::ValueString& tag_name, + const YAML::Node& node) const { + return cel::YamlPolicyParser::ParsePolicyTag(ctx, tag_name, node); + } + + absl::Status ParseRule(cel::CelPolicyParseContext& ctx, + const YAML::Node& node, + cel::Rule& rule) const override { + pybind11::gil_scoped_acquire gil; + pybind11::function override = pybind11::get_override(this, "parse_rule"); + if (override) { + override(pybind11::cast(&ctx, pybind11::return_value_policy::reference), + node, + pybind11::cast(&rule, pybind11::return_value_policy::reference)); + return absl::OkStatus(); + } + pybind11::gil_scoped_release gil_release; + return cel::YamlPolicyParser::ParseRule(ctx, node, rule); + } + + absl::Status SuperParseRule(cel::CelPolicyParseContext& ctx, + const YAML::Node& node, cel::Rule& rule) const { + return cel::YamlPolicyParser::ParseRule(ctx, node, rule); + } + + absl::StatusOr ParseRuleTag(cel::CelPolicyParseContext& ctx, + const cel::ValueString& tag_name, + const YAML::Node& node, + cel::Rule& rule) const override { + pybind11::gil_scoped_acquire gil; + pybind11::function override = + pybind11::get_override(this, "parse_rule_tag"); + if (override) { + auto py_result = override( + pybind11::cast(&ctx, pybind11::return_value_policy::reference), + tag_name, node, + pybind11::cast(&rule, pybind11::return_value_policy::reference)); + return py_result.cast(); + } + pybind11::gil_scoped_release gil_release; + return cel::YamlPolicyParser::ParseRuleTag(ctx, tag_name, node, rule); + } + + absl::StatusOr SuperParseRuleTag(cel::CelPolicyParseContext& ctx, + const cel::ValueString& tag_name, + const YAML::Node& node, + cel::Rule& rule) const { + return cel::YamlPolicyParser::ParseRuleTag(ctx, tag_name, node, rule); + } + + absl::StatusOr ParseMatch(cel::CelPolicyParseContext& ctx, + const YAML::Node& node, + cel::Rule& rule) const override { + pybind11::gil_scoped_acquire gil; + pybind11::function override = pybind11::get_override(this, "parse_match"); + if (override) { + auto py_result = override( + pybind11::cast(&ctx, pybind11::return_value_policy::reference), node, + pybind11::cast(&rule, pybind11::return_value_policy::reference)); + return py_result.cast(); + } + pybind11::gil_scoped_release gil_release; + return cel::YamlPolicyParser::ParseMatch(ctx, node, rule); + } + + absl::StatusOr SuperParseMatch(cel::CelPolicyParseContext& ctx, + const YAML::Node& node, + cel::Rule& rule) const { + return cel::YamlPolicyParser::ParseMatch(ctx, node, rule); + } + + absl::StatusOr ParseMatchTag(cel::CelPolicyParseContext& ctx, + const cel::ValueString& tag_name, + const YAML::Node& node, cel::Match& match, + cel::Rule& rule) const override { + pybind11::gil_scoped_acquire gil; + pybind11::function override = + pybind11::get_override(this, "parse_match_tag"); + if (override) { + auto py_result = override( + pybind11::cast(&ctx, pybind11::return_value_policy::reference), + tag_name, node, + pybind11::cast(&match, pybind11::return_value_policy::reference), + pybind11::cast(&rule, pybind11::return_value_policy::reference)); + return py_result.cast(); + } + pybind11::gil_scoped_release gil_release; + return cel::YamlPolicyParser::ParseMatchTag(ctx, tag_name, node, match, + rule); + } + + absl::StatusOr SuperParseMatchTag(cel::CelPolicyParseContext& ctx, + const cel::ValueString& tag_name, + const YAML::Node& node, + cel::Match& match, + cel::Rule& rule) const { + return cel::YamlPolicyParser::ParseMatchTag(ctx, tag_name, node, match, + rule); + } + + absl::StatusOr ParseVariable(cel::CelPolicyParseContext& ctx, + const YAML::Node& node, + cel::Rule& rule) const override { + pybind11::gil_scoped_acquire gil; + pybind11::function override = + pybind11::get_override(this, "parse_variable"); + if (override) { + auto py_result = override( + pybind11::cast(&ctx, pybind11::return_value_policy::reference), node, + pybind11::cast(&rule, pybind11::return_value_policy::reference)); + return py_result.cast(); + } + pybind11::gil_scoped_release gil_release; + return cel::YamlPolicyParser::ParseVariable(ctx, node, rule); + } + + absl::StatusOr SuperParseVariable( + cel::CelPolicyParseContext& ctx, const YAML::Node& node, + cel::Rule& rule) const { + return cel::YamlPolicyParser::ParseVariable(ctx, node, rule); + } + + absl::StatusOr ParseVariableTag( + cel::CelPolicyParseContext& ctx, const cel::ValueString& tag_name, + const YAML::Node& node, cel::Variable& variable) const override { + pybind11::gil_scoped_acquire gil; + pybind11::function override = + pybind11::get_override(this, "parse_variable_tag"); + if (override) { + auto py_result = override( + pybind11::cast(&ctx, pybind11::return_value_policy::reference), + tag_name, node, + pybind11::cast(&variable, pybind11::return_value_policy::reference)); + return py_result.cast(); + } + pybind11::gil_scoped_release gil_release; + return cel::YamlPolicyParser::ParseVariableTag(ctx, tag_name, node, + variable); + } + + absl::StatusOr SuperParseVariableTag(cel::CelPolicyParseContext& ctx, + const cel::ValueString& tag_name, + const YAML::Node& node, + cel::Variable& variable) const { + return cel::YamlPolicyParser::ParseVariableTag(ctx, tag_name, node, + variable); + } +}; + +// Memory safety with YAML::Node across C++ and Python boundaries is guaranteed +// by yaml-cpp's internal shared reference counting: +// 1. Shared Memory Arena: Every YAML::Node references a shared memory holder +// (`detail::shared_memory_holder`) that manages the underlying document +// nodes. +// 2. Automatic Reference Counting: Slices or individual node references stored +// in Python hold a strong reference to this memory arena, preventing +// premature deallocation even after the C++ parse session finishes. +// 3. Garbage Collection: Once all Python references are released, Python's +// garbage collector invokes the C++ destructor, which drops the reference +// count and safely reclaims the memory. +static void DefineYamlNodeBindings(pybind11::module& m) { + // Force the linker to include YAML::Node destructor symbols in static + // linkage. + if (m.attr("__name__").cast() == "___never___") { + YAML::Node n = YAML::Load("[]"); + } + py::class_(m, "YAMLNode") + .def(py::init<>()) + .def("is_scalar", [](const YAML::Node& node) { return node.IsScalar(); }) + .def("is_sequence", + [](const YAML::Node& node) { return node.IsSequence(); }) + .def("is_map", [](const YAML::Node& node) { return node.IsMap(); }) + .def("__len__", [](const YAML::Node& node) { return node.size(); }) + .def("scalar", [](const YAML::Node& node) { return node.Scalar(); }) + .def("__getitem__", + [](const YAML::Node& node, int index) { + if (index < 0 || static_cast(index) >= node.size()) { + throw py::index_error(); + } + return node[index]; + }) + .def("__getitem__", [](const YAML::Node& node, const std::string& key) { + return node[key]; + }); +} + +static void DefineCelPolicyParseContextBindings(pybind11::module& m) { + py::class_(m, "CelPolicyParseContext") + .def_property_readonly( + "policy", + [](cel::CelPolicyParseContext& self) -> cel::CelPolicy& { + return self.policy(); + }, + py::return_value_policy::reference_internal) + .def("report_error", + [](cel::CelPolicyParseContext& self, cel::CelPolicyElementId id, + const std::string& message) { self.ReportError(id, message); }) + .def("collect_metadata", + [](cel::CelPolicyParseContext& self, const YAML::Node& node) { + cel::CelPolicyElementId id = self.next_element_id(); + self.policy_source().NoteSourcePosition(id, node.Mark().pos); + return id; + }); +} + +static void DefineYamlPolicyParserBindings(pybind11::module& m) { + py::class_( + m, "CelPolicyYamlParser") + .def(py::init<>()) + .def("parse", + [](const cel::YamlPolicyParser& self, + absl::string_view yaml_content) { + auto source = ThrowIfError(cel::NewSource(yaml_content)); + auto policy_source = + std::make_shared(std::move(source)); + cel::CelPolicyParseResult result = [&]() { + py::gil_scoped_release gil_release; + return ThrowIfError( + cel::ParseYamlCelPolicy(policy_source, self)); + }(); + return result; + }) + .def( + "parse_policy_tag", + [](const cel::YamlPolicyParser& self, cel::CelPolicyParseContext& ctx, + const cel::ValueString& tag_name, const YAML::Node& node) { + return ThrowIfError(static_cast(self) + .SuperParsePolicyTag(ctx, tag_name, node)); + }) + .def( + "parse_rule", + [](const cel::YamlPolicyParser& self, cel::CelPolicyParseContext& ctx, + const YAML::Node& node, cel::Rule& rule) { + auto status = + static_cast(self).SuperParseRule( + ctx, node, rule); + if (!status.ok()) ThrowIfError(absl::StatusOr(status)); + }) + .def("parse_rule_tag", + [](const cel::YamlPolicyParser& self, + cel::CelPolicyParseContext& ctx, const cel::ValueString& tag_name, + const YAML::Node& node, cel::Rule& rule) { + return ThrowIfError( + static_cast(self) + .SuperParseRuleTag(ctx, tag_name, node, rule)); + }) + .def( + "parse_match", + [](const cel::YamlPolicyParser& self, cel::CelPolicyParseContext& ctx, + const YAML::Node& node, cel::Rule& rule) { + return ThrowIfError( + static_cast(self).SuperParseMatch( + ctx, node, rule)); + }) + .def("parse_match_tag", + [](const cel::YamlPolicyParser& self, + cel::CelPolicyParseContext& ctx, const cel::ValueString& tag_name, + const YAML::Node& node, cel::Match& match, cel::Rule& rule) { + return ThrowIfError( + static_cast(self) + .SuperParseMatchTag(ctx, tag_name, node, match, rule)); + }) + .def( + "parse_variable", + [](const cel::YamlPolicyParser& self, cel::CelPolicyParseContext& ctx, + const YAML::Node& node, cel::Rule& rule) { + return ThrowIfError(static_cast(self) + .SuperParseVariable(ctx, node, rule)); + }) + .def("parse_variable_tag", + [](const cel::YamlPolicyParser& self, + cel::CelPolicyParseContext& ctx, const cel::ValueString& tag_name, + const YAML::Node& node, cel::Variable& variable) { + return ThrowIfError( + static_cast(self) + .SuperParseVariableTag(ctx, tag_name, node, variable)); + }) + .def( + "get_value_string", + [](const cel::YamlPolicyParser& self, cel::CelPolicyParseContext& ctx, + const YAML::Node& node, const std::string& error_message) { + return self.GetValueString(ctx, node, error_message); + }); +} + +void DefineCelPolicyYamlParserBindings(pybind11::module& m) { + DefineYamlNodeBindings(m); + DefineCelPolicyParseContextBindings(m); + DefineYamlPolicyParserBindings(m); + + m.def( + "parse_yaml_cel_policy", + [](absl::string_view yaml_content, const cel::YamlPolicyParser* parser, + std::optional description) { + auto source = ThrowIfError( + cel::NewSource(yaml_content, description.value_or(""))); + auto policy_source = + std::make_shared(std::move(source)); + if (parser) { + return ThrowIfError(cel::ParseYamlCelPolicy(policy_source, *parser)); + } else { + return ThrowIfError(cel::ParseYamlCelPolicy(policy_source)); + } + }, + py::arg("yaml_content"), py::arg("parser") = py::none(), + py::arg("description") = py::none()); +} + +} // namespace cel_python diff --git a/cel_expr_python/policy/py_cel_policy_yaml_parser.h b/cel_expr_python/policy/py_cel_policy_yaml_parser.h new file mode 100644 index 0000000..0d09be4 --- /dev/null +++ b/cel_expr_python/policy/py_cel_policy_yaml_parser.h @@ -0,0 +1,26 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef THIRD_PARTY_CEL_PYTHON_POLICY_PY_CEL_POLICY_YAML_PARSER_H_ +#define THIRD_PARTY_CEL_PYTHON_POLICY_PY_CEL_POLICY_YAML_PARSER_H_ + +#include + +namespace cel_python { + +void DefineCelPolicyYamlParserBindings(pybind11::module& m); + +} // namespace cel_python + +#endif // THIRD_PARTY_CEL_PYTHON_POLICY_PY_CEL_POLICY_YAML_PARSER_H_ diff --git a/cel_expr_python/policy/testdata/cel_policy.yaml b/cel_expr_python/policy/testdata/cel_policy.yaml new file mode 100644 index 0000000..380faef --- /dev/null +++ b/cel_expr_python/policy/testdata/cel_policy.yaml @@ -0,0 +1,35 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +name: cel_policy +description: A test policy for CEL +display_name: Cel Policy +imports: +- name: cel.expr.conformance.proto3.TestAllTypes +- name: cel.expr.conformance.proto3.TestAllTypes.NestedEnum +rule: + id: test_rule + description: test rule description + variables: + - name: test_var + expression: > + TestAllTypes{single_int64: 10}.single_int64 + match: + - condition: > + spec.single_int32 > TestAllTypes{single_int64: 10}.single_int64 + output: | + "invalid spec, got single_int32=" + string(spec.single_int32) + ", wanted <= 10" + - condition: > + spec.standalone_enum == NestedEnum.BAR + output: | + "invalid spec, reference to BAR is not allowed" diff --git a/cel_expr_python/policy/testdata/cel_policy_invalid.yaml b/cel_expr_python/policy/testdata/cel_policy_invalid.yaml new file mode 100644 index 0000000..6a7beee --- /dev/null +++ b/cel_expr_python/policy/testdata/cel_policy_invalid.yaml @@ -0,0 +1,19 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +name: cel_policy_invalid +rule: + id: test_rule + match: + - condition: 10 + output: '"error"' diff --git a/cel_expr_python/policy/testdata/custom_policy_format.yaml b/cel_expr_python/policy/testdata/custom_policy_format.yaml new file mode 100644 index 0000000..20cd283 --- /dev/null +++ b/cel_expr_python/policy/testdata/custom_policy_format.yaml @@ -0,0 +1,32 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +name: cel_policy_custom_tags +description: A custom policy format +imports: +- name: cel.expr.conformance.proto3.TestAllTypes +purpose: test +version: 42 +inputs: +- name: test_input + assign: "42" +conditions: +- if: spec.single_string == "none" + then: "'zero'" + else: + if: spec.single_string == "integer" + then: + if: spec.single_int32 > 0 + then: "'positive integer'" + else: "'negative integer'" + else: "'not an integer'" diff --git a/cel_expr_python/policy/testdata/custom_policy_format_with_errors.yaml b/cel_expr_python/policy/testdata/custom_policy_format_with_errors.yaml new file mode 100644 index 0000000..84b628a --- /dev/null +++ b/cel_expr_python/policy/testdata/custom_policy_format_with_errors.yaml @@ -0,0 +1,32 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +name: cel_policy_custom_tags +description: A custom policy format +imports: +- name: cel.expr.conformance.proto3.TestAllTypes +purpose: + - testing +version: new +conditions: +- if: + spec.single_string: "none" + then: "'zero'" + else: "'not zero'" +- if: spec.single_string == "number" + then: + if: spec.single_int32 > 0 + then: "'positive integer'" + else: + - ignore +- else: "'negative integer'" diff --git a/cel_expr_python/py_cel_activation.cc b/cel_expr_python/py_cel_activation.cc index ef7edd2..538ef49 100644 --- a/cel_expr_python/py_cel_activation.cc +++ b/cel_expr_python/py_cel_activation.cc @@ -67,7 +67,12 @@ PyCelActivation::PyCelActivation( for (const auto& function : functions) { std::vector parameters; for (const auto& parameter : function->parameters()) { - parameters.push_back(parameter.GetKind()); + if (parameter.GetKind() == cel::Kind::kDyn || + parameter.GetKind() == cel::Kind::kTypeParam) { + parameters.push_back(cel::Kind::kAny); + } else { + parameters.push_back(parameter.GetKind()); + } } cel::FunctionDescriptor func_descriptor(function->function_name(), function->is_member(), parameters, diff --git a/cel_expr_python/py_cel_env.cc b/cel_expr_python/py_cel_env.cc index 6838bef..9a0a965 100644 --- a/cel_expr_python/py_cel_env.cc +++ b/cel_expr_python/py_cel_env.cc @@ -26,6 +26,8 @@ #include "absl/log/absl_check.h" #include "common/container.h" +#include "compiler/compiler.h" +#include "cel_expr_python/cel_compiler_capsule.h" #include "cel_expr_python/py_cel_activation.h" #include "cel_expr_python/py_cel_arena.h" #include "cel_expr_python/py_cel_env_config.h" @@ -145,6 +147,19 @@ void PyCelEnv::DefinePythonBindings(pybind11::module& m) { .def("compile", &PyCelEnv::Compile, py::arg("expression"), py::arg("disable_check") = false) .def("deserialize", &PyCelEnv::Deserialize, py::arg("serialized")) + .def("_compiler_capsule", + [](PyCelEnv& self) { + std::shared_ptr env = self.GetEnv(); + const cel::Compiler* compiler = nullptr; + { + py::gil_scoped_release gil_release; + compiler = ThrowIfError(env->GetCompiler()); + } + // Aliasing constructor: the capsule keeps `env`, which owns the + // compiler, alive. + return MakeCelCompilerCapsule( + std::shared_ptr(env, compiler)); + }) .def( "Activation", [](PyCelEnv& self, diff --git a/cel_expr_python/py_cel_env_internal.cc b/cel_expr_python/py_cel_env_internal.cc index 7983bfe..f2dab0d 100644 --- a/cel_expr_python/py_cel_env_internal.cc +++ b/cel_expr_python/py_cel_env_internal.cc @@ -322,7 +322,11 @@ absl::StatusOr> PyCelEnvInternal::BuildRuntime( CEL_PYTHON_ASSIGN_OR_RETURN( cel::Type type, cel::TypeInfoToType(parameter, descriptor_pool_.get(), &arena)); - param_kinds.push_back(static_cast(type.kind())); + cel::Kind kind = static_cast(type.kind()); + if (kind == cel::Kind::kDyn || kind == cel::Kind::kTypeParam) { + kind = cel::Kind::kAny; + } + param_kinds.push_back(kind); } cel::FunctionDescriptor descriptor( function_config.name, overload_config.is_member_function, param_kinds, diff --git a/cel_expr_python/py_cel_function.cc b/cel_expr_python/py_cel_function.cc index f42d38c..6c5a315 100644 --- a/cel_expr_python/py_cel_function.cc +++ b/cel_expr_python/py_cel_function.cc @@ -16,6 +16,7 @@ #include +#include #include #include #include @@ -107,14 +108,34 @@ absl::StatusOr PyCelFunctionAdapter::Invoke( CEL_PYTHON_ASSIGN_OR_RETURN(auto py_arena, PyCelArena::FromProtoArena(context.arena())); PyObject* py_args = PyTuple_New(args.size()); - for (int i = 0; i < args.size(); ++i) { - PyTuple_SetItem(py_args, i, - CelValueToPyObject(args[i], env, py_arena, - /*plain_value=*/true)); + if (py_args == nullptr) { + absl::Status status = PyErr_toStatus(); + if (status.ok()) { + status = absl::ResourceExhaustedError("Failed to allocate args tuple"); + } + return cel::ErrorValue::From(status, context.arena()); + } + for (size_t i = 0; i < args.size(); ++i) { + PyObject* py_arg = CelValueToPyObject(args[i], env, py_arena, + /*plain_value=*/true); + if (py_arg == nullptr) { + absl::Status status = PyErr_toStatus(); + Py_DECREF(py_args); + if (status.ok()) { + status = absl::InternalError( + "Failed to convert CEL argument to Python object"); + } + return cel::ErrorValue::From(status, context.arena()); + } + PyTuple_SetItem(py_args, i, py_arg); } PyObject* result = PyObject_CallObject(py_function_, py_args); - Py_DECREF(py_args); absl::Status status = PyErr_toStatus(); + Py_DECREF(py_args); + if (status.ok() && result == nullptr) { + status = absl::InternalError( + "Python function returned null without setting an error"); + } if (!status.ok()) { Py_XDECREF(result); return cel::ErrorValue::From(status, context.arena()); @@ -123,8 +144,15 @@ absl::StatusOr PyCelFunctionAdapter::Invoke( absl::StatusOr cel_result = PyObjectToCelValue( result, return_type_, [this]() { - return absl::StrFormat("Python function '%s'", - PyUnicode_AsUTF8(PyObject_Repr(py_function_))); + PyObject* repr = PyObject_Repr(py_function_); + const char* repr_cstr = + repr != nullptr ? PyUnicode_AsUTF8(repr) : nullptr; + if (repr_cstr == nullptr) { + PyErr_Clear(); + } + std::string repr_str = repr_cstr != nullptr ? repr_cstr : ""; + Py_XDECREF(repr); + return absl::StrFormat("Python function '%s'", repr_str); }, env, context.arena()); Py_XDECREF(result); diff --git a/cel_expr_python/py_cel_module.cc b/cel_expr_python/py_cel_module.cc index 8879e00..ea4075e 100644 --- a/cel_expr_python/py_cel_module.cc +++ b/cel_expr_python/py_cel_module.cc @@ -19,6 +19,7 @@ #include "cel_expr_python/py_cel_expression.h" #include "cel_expr_python/py_cel_function.h" #include "cel_expr_python/py_cel_function_decl.h" +#include "cel_expr_python/py_cel_optional.h" #include "cel_expr_python/py_cel_options.h" #include "cel_expr_python/py_cel_overload.h" #include "cel_expr_python/py_cel_python_extension.h" @@ -35,6 +36,7 @@ PYBIND11_MODULE(cel, m, pybind11::mod_gil_not_used()) { PyCelArena::DefinePythonBindings(m); PyCelType::DefinePythonBindings(m); + PyCelOptional::DefinePythonBindings(m); PyCelValue::DefinePythonBindings(m); PyCelActivation::DefinePythonBindings(m); PyCelExpression::DefinePythonBindings(m); diff --git a/cel_expr_python/py_cel_optional.cc b/cel_expr_python/py_cel_optional.cc new file mode 100644 index 0000000..6cc4e33 --- /dev/null +++ b/cel_expr_python/py_cel_optional.cc @@ -0,0 +1,130 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "cel_expr_python/py_cel_optional.h" + +#include // IWYU pragma: keep - Needed for PyObject + +#include + +#include "absl/log/absl_check.h" +#include "absl/strings/str_cat.h" +#include + +namespace cel_python { + +namespace py = ::pybind11; + +void PyCelOptional::DefinePythonBindings(py::module& m) { + py::class_(m, "Optional") + .def(py::init<>()) + .def_static( + "of", [](py::object value) { return PyCelOptional::Of(value); }, + py::arg("value")) + .def_static("none", &PyCelOptional::None) + .def("has_value", &PyCelOptional::HasValue) + .def("value", &PyCelOptional::Value) + .def("or_value", &PyCelOptional::OrValue, py::arg("default")) + .def("__bool__", &PyCelOptional::HasValue) + .def("__eq__", &PyCelOptional::Equals, py::is_operator()) + .def("__repr__", &PyCelOptional::ToString) + .def_static("__class_getitem__", &PyCelOptional::ClassGetItem, + py::arg("item")); +} + +PyCelOptional::PyCelOptional(PyObject* value) : value_(value) { + if (value_ != nullptr) { + ABSL_CHECK(PyGILState_Check()); + Py_INCREF(value_); + } +} + +PyCelOptional::PyCelOptional(PyCelOptional&& other) noexcept + : value_(other.value_) { + other.value_ = nullptr; +} + +PyCelOptional::~PyCelOptional() { + if (value_ != nullptr) { + if (!PyGILState_Check()) { + py::gil_scoped_acquire acquire; + Py_DECREF(value_); + } else { + Py_DECREF(value_); + } + } +} + +py::object PyCelOptional::Equals(py::handle other) const { + ABSL_CHECK(PyGILState_Check()); + if (!py::isinstance(other)) { + return py::reinterpret_borrow(Py_NotImplemented); + } + const PyCelOptional& other_opt = other.cast(); + if (value_ == nullptr && other_opt.value_ == nullptr) { + return py::bool_(true); + } + if (value_ == nullptr || other_opt.value_ == nullptr) { + return py::bool_(false); + } + PyObject* res = PyObject_RichCompare(value_, other_opt.value_, Py_EQ); + if (res == nullptr) { + throw py::error_already_set(); + } + return py::reinterpret_steal(res); +} + +PyCelOptional PyCelOptional::Of(py::handle value) { + ABSL_CHECK(value.ptr() != nullptr); + return PyCelOptional(value.ptr()); +} + +// Enables runtime subscripting (e.g. `cel.Optional[int]`) for type +// annotations on Python < 3.14 and in runtime type-hint inspection. +py::object PyCelOptional::ClassGetItem(py::handle item) { + ABSL_CHECK(PyGILState_Check()); + PyObject* alias = + Py_GenericAlias(py::type::of().ptr(), item.ptr()); + if (alias == nullptr) { + throw py::error_already_set(); + } + return py::reinterpret_steal(alias); +} + +py::object PyCelOptional::Value() const { + ABSL_CHECK(PyGILState_Check()); + if (value_ == nullptr) { + throw py::value_error("optional.none() dereference"); + } + return py::reinterpret_borrow(value_); +} + +py::object PyCelOptional::OrValue(py::object default_value) const { + ABSL_CHECK(PyGILState_Check()); + if (value_ == nullptr) { + return default_value; + } + return py::reinterpret_borrow(value_); +} + +std::string PyCelOptional::ToString() const { + ABSL_CHECK(PyGILState_Check()); + if (value_ == nullptr) { + return "optional.none()"; + } + py::str repr_str = py::repr(py::reinterpret_borrow(value_)); + return absl::StrCat("optional.of(", static_cast(repr_str), ")"); +} + +} // namespace cel_python diff --git a/cel_expr_python/py_cel_optional.h b/cel_expr_python/py_cel_optional.h new file mode 100644 index 0000000..aea4037 --- /dev/null +++ b/cel_expr_python/py_cel_optional.h @@ -0,0 +1,65 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef THIRD_PARTY_CEL_PYTHON_PY_CEL_OPTIONAL_H_ +#define THIRD_PARTY_CEL_PYTHON_PY_CEL_OPTIONAL_H_ + +#include // IWYU pragma: keep - Needed for PyObject + +#include + +#include + +namespace cel_python { + +// Represents a CEL optional value in Python (`optional.of(v)` or +// `optional.none()`). +// +// Internally, `value_ == nullptr` represents `optional.none()`, whereas a +// non-null `value_` (including `Py_None` for `optional.of(null)`) represents a +// present value with an owned Python reference. +class PyCelOptional { + public: + static void DefinePythonBindings(pybind11::module& m); + + // Creates an empty optional (`optional.none()`). + PyCelOptional() : value_(nullptr) {} + + static PyCelOptional None() { return PyCelOptional(); } + static PyCelOptional Of(pybind11::handle value); + static pybind11::object ClassGetItem(pybind11::handle item); + + PyCelOptional(const PyCelOptional& other) = delete; + PyCelOptional(PyCelOptional&& other) noexcept; + PyCelOptional& operator=(const PyCelOptional& other) = delete; + PyCelOptional& operator=(PyCelOptional&& other) = delete; + ~PyCelOptional(); + + bool HasValue() const { return value_ != nullptr; } + PyObject* RawValue() const { return value_; } + + pybind11::object Value() const; + pybind11::object OrValue(pybind11::object default_value) const; + pybind11::object Equals(pybind11::handle other) const; + std::string ToString() const; + + private: + explicit PyCelOptional(PyObject* value); + + PyObject* value_; +}; + +} // namespace cel_python + +#endif // THIRD_PARTY_CEL_PYTHON_PY_CEL_OPTIONAL_H_ diff --git a/cel_expr_python/py_cel_python_extension.cc b/cel_expr_python/py_cel_python_extension.cc index 6ff92ac..50019fc 100644 --- a/cel_expr_python/py_cel_python_extension.cc +++ b/cel_expr_python/py_cel_python_extension.cc @@ -117,7 +117,8 @@ absl::Status PyCelPythonExtension::ConfigureRuntime( std::vector types; types.reserve(overload.parameters().size()); for (const PyCelType& arg : overload.parameters()) { - if (arg.GetKind() == cel::Kind::kDyn) { + if (arg.GetKind() == cel::Kind::kDyn || + arg.GetKind() == cel::Kind::kTypeParam) { // C++ runtime dispatcher historically uses kAny for wildcard type ( // not distinguishing between dyn and any) types.push_back(cel::Kind::kAny); diff --git a/cel_expr_python/py_cel_type.cc b/cel_expr_python/py_cel_type.cc index eb4bf9a..1ff2dd4 100644 --- a/cel_expr_python/py_cel_type.cc +++ b/cel_expr_python/py_cel_type.cc @@ -14,14 +14,17 @@ #include "cel_expr_python/py_cel_type.h" -#include // IWYU pragma: keep - Needed for PyObject +#include // IWYU pragma: keep - Needed for PyObject +#include // IWYU pragma: keep +#include #include #include #include #include #include +#include "absl/base/call_once.h" #include "absl/base/no_destructor.h" #include "absl/container/flat_hash_map.h" #include "absl/log/absl_check.h" @@ -40,6 +43,7 @@ #include "common/value_kind.h" #include "env/config.h" #include "env/type_info.h" +#include "cel_expr_python/py_cel_optional.h" #include "cel_expr_python/py_error_status.h" #include "cel_expr_python/status_macros.h" #include "google/protobuf/arena.h" @@ -70,6 +74,8 @@ void PyCelType::DefinePythonBindings(py::module& m) { type.def_static("List", &PyCelType::ListType, py::arg("element_type")) .def_static("Map", &PyCelType::MapType, py::arg("key_type"), py::arg("element_type")) + .def_static("Optional", &PyCelType::OptionalType, + py::arg("element_type") = PyCelType::Dyn()) .def_static("Type", &PyCelType::TypeType, py::arg("element_type")) .def_static("AbstractType", &PyCelType::AbstractType, py::arg("name"), py::arg("params") = std::vector()); @@ -86,6 +92,7 @@ void PyCelType::DefinePythonBindings(py::module& m) { type.attr("ERROR") = PyCelType::Error(); type.attr("LIST") = PyCelType::List(); type.attr("MAP") = PyCelType::Map(); + type.attr("OPTIONAL") = PyCelType::Optional(); type.attr("TIMESTAMP") = PyCelType::Timestamp(); type.attr("DURATION") = PyCelType::Duration(); type.attr("TYPE") = PyCelType::Type(); @@ -102,7 +109,7 @@ PyCelType::PyCelType(const std::string& name) : parameters_({}) { name_ = "DURATION"; } else { kind_ = cel::Kind::kMessage; - name_ = std::move(name); + name_ = name; } } @@ -131,13 +138,17 @@ PyCelType PyCelType::MapType(const PyCelType& key_type, return PyCelType(cel::Kind::kMap, "MAP", {key_type, value_type}); } +PyCelType PyCelType::OptionalType(const PyCelType& element_type) { + return AbstractType("optional_type", {element_type}); +} + PyCelType PyCelType::TypeType(const PyCelType& parameter) { return PyCelType(cel::Kind::kType, "TYPE", {parameter}); } PyCelType PyCelType::AbstractType(const std::string& name, const std::vector& params) { - return PyCelType(cel::Kind::kType, name, params); + return PyCelType(cel::Kind::kOpaque, name, params); } bool PyCelType::operator==(const PyCelType& other) const { @@ -146,14 +157,22 @@ bool PyCelType::operator==(const PyCelType& other) const { } bool PyCelType::IsAssignableFrom(const PyCelType& other) const { + if (kind_ == cel::Kind::kDyn) { + return true; + } + if (kind_ != other.kind_) { return false; } - if (kind_ == cel::Kind::kMessage) { + if (kind_ == cel::Kind::kMessage || kind_ == cel::Kind::kTypeParam) { return name_ == other.name_; } + if (kind_ == cel::Kind::kOpaque && name_ != other.name_) { + return false; + } + if (parameters_.empty()) { return true; } @@ -162,7 +181,7 @@ bool PyCelType::IsAssignableFrom(const PyCelType& other) const { return false; } - for (int i = 0; i < parameters_.size(); ++i) { + for (size_t i = 0; i < parameters_.size(); ++i) { if (!parameters_[i].IsAssignableFrom(other.parameters_[i])) { return false; } @@ -245,6 +264,13 @@ const PyCelType& PyCelType::Map() { return *kInternalType; } +// cel.Type.Optional(cel.Type.DYN) aka cel.Type.OPTIONAL +const PyCelType& PyCelType::Optional() { + static const absl::NoDestructor kInternalType( + PyCelType::OptionalType(PyCelType::Dyn())); + return *kInternalType; +} + const PyCelType& PyCelType::Timestamp() { static const absl::NoDestructor kInternalType( cel::Kind::kTimestamp, "TIMESTAMP"); @@ -336,6 +362,9 @@ PyCelType PyCelType::FromCelType(const cel::Type& cel_type) { } case cel::TypeKind::kDyn: return PyCelType::Dyn(); + case cel::TypeKind::kTypeParam: + return PyCelType(cel::Kind::kTypeParam, + std::string(cel_type.AsTypeParam()->name())); case cel::TypeKind::kOpaque: { const auto& abstract_type = cel_type.AsOpaque(); std::vector params; @@ -395,15 +424,25 @@ PyCelType PyCelType::ForCelValue(const cel::Value& cel_value) { return PyCelType::Unknown(); case cel::ValueKind::kType: return PyCelType::Type(); + case cel::ValueKind::kOpaque: + return FromCelType(cel_value.GetOpaque().GetRuntimeType()); default: // TODO(b/447177883): Add support for all standard types. return PyCelType::Error(); } } +static void EnsureDateTimeModuleImported() { + static absl::once_flag import_py_datetime_once_flag; + absl::call_once(import_py_datetime_once_flag, []() { PyDateTime_IMPORT; }); +} + absl::StatusOr PyCelType::ForPyObject( PyObject* py_object, const std::function& context) { ABSL_CHECK(PyGILState_Check()); + if (py::isinstance(py::handle(py_object))) { + return PyCelType::Optional(); + } PyTypeObject* py_type = Py_TYPE(py_object); const absl::flat_hash_map& primitive_py_types = GetPrimitivePyTypes(); @@ -450,6 +489,18 @@ absl::StatusOr PyCelType::ForPyObject( PyErr_Clear(); } + EnsureDateTimeModuleImported(); + if (PyDateTime_Check(py_object)) { + return PyCelType::Timestamp(); + } + if (PyDelta_Check(py_object)) { + return PyCelType::Duration(); + } + + if (py::isinstance(py::handle(py_object))) { + return PyCelType::Type(); + } + PyTypeObject* type = Py_TYPE(py_object); return absl::InvalidArgumentError(absl::StrFormat( "Non-CEL value type for '%s': %s.", context(), type->tp_name)); @@ -509,6 +560,8 @@ PyCelType PyCelType::FromTypeProto(const cel::expr::Type& type) { } case cel::expr::Type::kMessageType: return PyCelType(type.message_type()); + case cel::expr::Type::kTypeParam: + return PyCelType(cel::Kind::kTypeParam, type.type_param()); case cel::expr::Type::kType: { cel::expr::Type parameter = type.type(); if (parameter.type_kind_case()) { @@ -537,6 +590,9 @@ PyCelType PyCelType::FromTypeProto(const cel::expr::Type& type) { } PyCelType PyCelType::FromTypeInfo(const cel::Config::TypeInfo& type_info) { + if (type_info.is_type_param) { + return PyCelType(cel::Kind::kTypeParam, type_info.name); + } if (type_info.name == "null") return PyCelType::Null(); if (type_info.name == "bool") return PyCelType::Bool(); if (type_info.name == "int") return PyCelType::Int(); @@ -560,13 +616,19 @@ PyCelType PyCelType::FromTypeInfo(const cel::Config::TypeInfo& type_info) { return PyCelType::MapType(FromTypeInfo(type_info.params[0]), FromTypeInfo(type_info.params[1])); } + if (type_info.name == "optional_type") { + if (type_info.params.empty()) { + return PyCelType::Optional(); + } + return PyCelType::OptionalType(FromTypeInfo(type_info.params[0])); + } if (type_info.name == "type") { if (type_info.params.empty()) { return PyCelType::Type(); } return PyCelType::TypeType(FromTypeInfo(type_info.params[0])); } - if (type_info.is_type_param || !type_info.params.empty()) { + if (!type_info.params.empty()) { std::vector params; params.reserve(type_info.params.size()); for (const auto& param : type_info.params) { @@ -581,6 +643,8 @@ absl::StatusOr PyCelType::ToCelType( const PyCelType& type, google::protobuf::Arena* arena, const google::protobuf::DescriptorPool& descriptor_pool) { switch (type.GetKind()) { + case cel::Kind::kNull: + return cel::NullType(); case cel::Kind::kBool: return cel::BoolType(); case cel::Kind::kInt: @@ -623,6 +687,36 @@ absl::StatusOr PyCelType::ToCelType( return cel::TimestampType(); case cel::Kind::kDuration: return cel::DurationType(); + case cel::Kind::kType: { + if (type.GetParamCount() == 0) { + return cel::TypeType(); + } + CEL_PYTHON_ASSIGN_OR_RETURN( + cel::Type param_type, + ToCelType(type.GetParam(0), arena, descriptor_pool)); + return cel::TypeType(arena, param_type); + } + case cel::Kind::kOpaque: { + std::vector params; + params.reserve(type.GetParamCount()); + for (size_t i = 0; i < type.GetParamCount(); ++i) { + CEL_PYTHON_ASSIGN_OR_RETURN( + cel::Type param_type, + ToCelType(type.GetParam(i), arena, descriptor_pool)); + params.push_back(param_type); + } + if (type.name_ == cel::OptionalType::kName && params.size() == 1) { + return cel::OptionalType(arena, params[0]); + } + const std::string* arena_name = + google::protobuf::Arena::Create(arena, type.name_); + return cel::OpaqueType(arena, *arena_name, params); + } + case cel::Kind::kTypeParam: { + const std::string* arena_name = + google::protobuf::Arena::Create(arena, type.name_); + return cel::TypeParamType(*arena_name); + } default: return cel::DynType(); } @@ -674,11 +768,21 @@ cel::Config::TypeInfo PyCelType::ToTypeInfo(const PyCelType& type) { case cel::Kind::kDuration: type_info.name = "duration"; break; + case cel::Kind::kType: + type_info.name = "type"; + if (!type.parameters_.empty()) { + type_info.params.push_back(ToTypeInfo(type.GetParam(0))); + } + break; case cel::Kind::kDyn: type_info.name = "dyn"; break; + case cel::Kind::kTypeParam: + type_info.name = type.name_; + type_info.is_type_param = true; + break; default: - type_info.name = type.GetName(); + type_info.name = type.name_; for (const PyCelType& param : type.parameters_) { type_info.params.push_back(ToTypeInfo(param)); } diff --git a/cel_expr_python/py_cel_type.h b/cel_expr_python/py_cel_type.h index 19cac0e..85c969c 100644 --- a/cel_expr_python/py_cel_type.h +++ b/cel_expr_python/py_cel_type.h @@ -58,6 +58,7 @@ class PyCelType { // May throw if the key type is not supported. static PyCelType MapType(const PyCelType& key_type, const PyCelType& value_type); + static PyCelType OptionalType(const PyCelType& element_type); static PyCelType TypeType(const PyCelType& parameter); static PyCelType AbstractType(const std::string& name, const std::vector& params = {}); @@ -66,6 +67,9 @@ class PyCelType { std::string GetName() const; bool IsAssignableFrom(const PyCelType& other) const; bool IsMessage() const { return kind_ == cel::Kind::kMessage; } + bool IsOptional() const { + return kind_ == cel::Kind::kOpaque && name_ == "optional_type"; + } size_t GetParamCount() const { return parameters_.size(); } const PyCelType& GetParam(int index) const { return parameters_[index]; } @@ -97,6 +101,7 @@ class PyCelType { static const PyCelType& Dyn(); static const PyCelType& List(); static const PyCelType& Map(); + static const PyCelType& Optional(); static const PyCelType& Timestamp(); static const PyCelType& Duration(); static const PyCelType& Type(); diff --git a/cel_expr_python/py_cel_value.cc b/cel_expr_python/py_cel_value.cc index ca13f77..ac732a6 100644 --- a/cel_expr_python/py_cel_value.cc +++ b/cel_expr_python/py_cel_value.cc @@ -40,6 +40,7 @@ #include "cel_expr_python/free_threading_mutex.h" #include "cel_expr_python/py_cel_arena.h" #include "cel_expr_python/py_cel_env_internal.h" +#include "cel_expr_python/py_cel_optional.h" #include "cel_expr_python/py_cel_type.h" #include "cel_expr_python/py_cel_value_provider.h" #include "cel_expr_python/py_error_status.h" @@ -55,6 +56,24 @@ namespace cel_python { namespace py = ::pybind11; +namespace { + +std::string PyObjectRepr(PyObject* obj) { + if (obj == nullptr) { + return ""; + } + PyObject* repr = PyObject_Repr(obj); + const char* repr_cstr = repr != nullptr ? PyUnicode_AsUTF8(repr) : nullptr; + if (repr_cstr == nullptr) { + PyErr_Clear(); + } + std::string repr_str = repr_cstr != nullptr ? repr_cstr : ""; + Py_XDECREF(repr); + return repr_str; +} + +} // namespace + void PyCelValue::DefinePythonBindings(py::module& m) { py::class_(m, "Value") .def("type", &PyCelValue::Type) @@ -461,8 +480,11 @@ PyObject* CelValueToPyObject(const cel::Value& cel_value, cel::ListValue list = *cel_value.AsList(); Py_ssize_t size = *list.Size(); PyObject* py_list = PyList_New(size); + if (py_list == nullptr) { + return nullptr; + } absl::Status status = absl::OkStatus(); - for (int i = 0; i < size; ++i) { + for (Py_ssize_t i = 0; i < size; ++i) { PyObject* py_item; if (plain_value) { absl::StatusOr item = @@ -479,6 +501,10 @@ PyObject* CelValueToPyObject(const cel::Value& cel_value, .release() .ptr(); } + if (py_item == nullptr) { + Py_DECREF(py_list); + return nullptr; + } // PyList_SetItem is special in that it "steals" the reference to the // item being set, so we must not decref it. // https://docs.python.org/3/c-api/list.html#:~:text=int-,pylist_setitem @@ -505,12 +531,16 @@ PyObject* CelValueToPyObject(const cel::Value& cel_value, } PyObject* py_map = PyDict_New(); + if (py_map == nullptr) { + return nullptr; + } Py_ssize_t size = *key_list.Size(); - for (int i = 0; i < size; ++i) { + for (Py_ssize_t i = 0; i < size; ++i) { absl::StatusOr key = key_list.Get(i, env->GetDescriptorPool(), env->GetMessageFactory(), arena->GetArena()); if (!key.ok()) { + Py_DECREF(py_map); PyErr_Format(PyExc_RuntimeError, "Cannot get key: %s", key.status().ToString().c_str()); return nullptr; @@ -518,6 +548,10 @@ PyObject* CelValueToPyObject(const cel::Value& cel_value, PyObject* py_key = CelValueToPyObject(*key, env, arena, /*plain_value=*/true); + if (py_key == nullptr) { + Py_DECREF(py_map); + return nullptr; + } PyObject* py_value; if (plain_value) { cel::Value element_value; @@ -525,9 +559,10 @@ PyObject* CelValueToPyObject(const cel::Value& cel_value, map.Get(*key, env->GetDescriptorPool(), env->GetMessageFactory(), arena->GetArena(), &element_value); if (!status.ok()) { + Py_DECREF(py_key); + Py_DECREF(py_map); PyErr_Format(PyExc_RuntimeError, "Cannot get value: %s", status.ToString().c_str()); - Py_DECREF(py_key); return nullptr; } py_value = CelValueToPyObject(element_value, env, arena, @@ -537,12 +572,21 @@ PyObject* CelValueToPyObject(const cel::Value& cel_value, .release() .ptr(); } + if (py_value == nullptr) { + Py_DECREF(py_key); + Py_DECREF(py_map); + return nullptr; + } // In contrast to PyList_SetItem, PyDict_SetItem does not "steal" the // reference to the key and value being set, so we must decref them. // https://docs.python.org/3/c-api/dict.html#:~:text=int-,pydict_setitem - PyDict_SetItem(py_map, py_key, py_value); + int set_status = PyDict_SetItem(py_map, py_key, py_value); Py_DECREF(py_key); Py_DECREF(py_value); + if (set_status < 0) { + Py_DECREF(py_map); + return nullptr; + } } return py_map; } @@ -570,6 +614,30 @@ PyObject* CelValueToPyObject(const cel::Value& cel_value, const cel::Type& type = cel_value.AsType()->type(); return py::cast(PyCelType::FromCelType(type)).release().ptr(); } + case cel::ValueKind::kOpaque: { + if (auto optional_value = cel_value.AsOptional(); + optional_value.has_value()) { + if (!optional_value->HasValue()) { + return py::cast(PyCelOptional::None()).release().ptr(); + } + cel::Value inner_value = optional_value->Value(); + if (plain_value) { + PyObject* py_inner = CelValueToPyObject(inner_value, env, arena, + /*plain_value=*/true); + if (py_inner == nullptr) { + return nullptr; + } + py::object inner_obj = py::reinterpret_steal(py_inner); + return py::cast(PyCelOptional::Of(inner_obj)).release().ptr(); + } + py::object inner_val_obj = + py::cast(PyCelValue(inner_value, arena, env)); + return py::cast(PyCelOptional::Of(inner_val_obj)).release().ptr(); + } + PyErr_Format(PyExc_RuntimeError, "Unsupported opaque type: %s", + std::string(cel_value.GetOpaque().GetTypeName()).c_str()); + return nullptr; + } default: { PyErr_Format(PyExc_RuntimeError, "Unimplemented type: %d", (int)cel_value.kind()); @@ -674,11 +742,13 @@ static cel::Value ProtoMessageToCelValue( return *wrapped_message; } +// This should be called with the GIL held. absl::StatusOr PyObjectToCelValue( PyObject* py_object, const PyCelType& expected_type, absl::FunctionRef context, const std::shared_ptr& env, google::protobuf::Arena* arena, bool bypass_type_check) { + ABSL_CHECK(PyGILState_Check()); if (!py_object) { return cel::ErrorValue::From( absl::InvalidArgumentError( @@ -688,7 +758,8 @@ absl::StatusOr PyObjectToCelValue( } switch (expected_type.GetKind()) { - case cel::Kind::kDyn: { + case cel::Kind::kDyn: + case cel::Kind::kTypeParam: { CEL_PYTHON_ASSIGN_OR_RETURN(const PyCelType& type, PyCelType::ForPyObject(py_object, context)); return PyObjectToCelValue(py_object, type, context, env, arena, @@ -767,9 +838,9 @@ absl::StatusOr PyObjectToCelValue( time = py::cast(py_object); } catch (const py::cast_error& e) { PyErr_Clear(); - return absl::UnknownError(absl::StrFormat( - "Cannot convert %s to absl::Time: %s", - PyUnicode_AsUTF8(PyObject_Repr(py_object)), e.what())); + return absl::UnknownError( + absl::StrFormat("Cannot convert %s to absl::Time: %s", + PyObjectRepr(py_object), e.what())); } absl::StatusOr timestamp_value = cel::SafeTimestampValue(time); @@ -808,7 +879,7 @@ absl::StatusOr PyObjectToCelValue( PyErr_Clear(); return absl::UnknownError( absl::StrFormat("Cannot convert %s to cel.Type.TIMESTAMP", - PyUnicode_AsUTF8(PyObject_Repr(py_object)))); + PyObjectRepr(py_object))); } absl::Time time = absl::FromUnixSeconds(seconds) + absl::Nanoseconds(nanos); @@ -830,9 +901,9 @@ absl::StatusOr PyObjectToCelValue( duration = py::cast(py_object); } catch (const py::cast_error& e) { PyErr_Clear(); - return absl::UnknownError(absl::StrFormat( - "Cannot convert %s to absl::Duration: %s", - PyUnicode_AsUTF8(PyObject_Repr(py_object)), e.what())); + return absl::UnknownError( + absl::StrFormat("Cannot convert %s to absl::Duration: %s", + PyObjectRepr(py_object), e.what())); } absl::StatusOr duration_value = cel::SafeDurationValue(duration); @@ -869,9 +940,8 @@ absl::StatusOr PyObjectToCelValue( Py_XDECREF(py_nanos); if (!ok) { PyErr_Clear(); - return absl::UnknownError( - absl::StrFormat("Cannot convert %s to absl::Duration", - PyUnicode_AsUTF8(PyObject_Repr(py_object)))); + return absl::UnknownError(absl::StrFormat( + "Cannot convert %s to absl::Duration", PyObjectRepr(py_object))); } absl::Duration duration = absl::Seconds(seconds) + absl::Nanoseconds(nanos); @@ -953,28 +1023,26 @@ absl::StatusOr PyObjectToCelValue( while (PyDict_Next(py_object, &pos, &key, &value)) { // Note: PyDict_Next returns borrowed references, so we shouldn't // DECREF them. - CEL_PYTHON_ASSIGN_OR_RETURN( - cel::Value converted_key, - PyObjectToCelValue( - key, key_type, - [context, key]() { - return absl::StrFormat( - "%s[%s]", context(), - PyUnicode_AsUTF8(PyObject_Repr(key))); - }, - env, arena)); - - CEL_PYTHON_ASSIGN_OR_RETURN( - cel::Value converted_value, - PyObjectToCelValue( - value, value_type, - [context, key, value]() { - return absl::StrFormat( - "%s[%s] -> %s", context(), - PyUnicode_AsUTF8(PyObject_Repr(key)), - PyUnicode_AsUTF8(PyObject_Repr(value))); - }, - env, arena)); + CEL_PYTHON_ASSIGN_OR_RETURN(cel::Value converted_key, + PyObjectToCelValue( + key, key_type, + [context, key]() { + return absl::StrFormat( + "%s[%s]", context(), + PyObjectRepr(key)); + }, + env, arena)); + + CEL_PYTHON_ASSIGN_OR_RETURN(cel::Value converted_value, + PyObjectToCelValue( + value, value_type, + [context, key, value]() { + return absl::StrFormat( + "%s[%s] -> %s", context(), + PyObjectRepr(key), + PyObjectRepr(value)); + }, + env, arena)); CEL_PYTHON_RETURN_IF_ERROR( builder->Put(converted_key, converted_value)); } @@ -990,6 +1058,59 @@ absl::StatusOr PyObjectToCelValue( } return InvalidTypeError(py_object, context, expected_type, arena); } + case cel::Kind::kType: { + if (py::isinstance(py::handle(py_object))) { + // Copy PyCelType while holding the GIL so it remains self-contained + // across GIL release. + PyCelType py_type = py::cast(py_object); + absl::StatusOr cel_type; + { + // Release the GIL before calling into C++ DescriptorPool to maintain + // the lock hierarchy (DescriptorPool Mutex -> Python GIL). + // PyCelType::ToCelType may resolve message types via DescriptorPool, + // which on a cache miss calls back into Python via + // PyDescriptorDatabase (re-acquiring the GIL on demand). + py::gil_scoped_release gil_release; + cel_type = + PyCelType::ToCelType(py_type, arena, *env->GetDescriptorPool()); + } + // The GIL is re-acquired here. Check if descriptor lookups raised a + // Python exception in PyDescriptorDatabase. + CEL_PYTHON_RETURN_IF_ERROR(PyErr_toStatus()); + CEL_PYTHON_RETURN_IF_ERROR(cel_type.status()); + return cel::TypeValue(*cel_type); + } + return InvalidTypeError(py_object, context, expected_type, arena); + } + case cel::Kind::kOpaque: { + if (expected_type.IsOptional()) { + if (!py::isinstance(py::handle(py_object))) { + return InvalidTypeError(py_object, context, expected_type, arena); + } + const PyCelOptional& py_opt = + py::handle(py_object).cast(); + if (!py_opt.HasValue()) { + return cel::OptionalValue::None(); + } + const PyCelType& element_type = expected_type.GetParamCount() > 0 + ? expected_type.GetParam(0) + : PyCelType::Dyn(); + CEL_PYTHON_ASSIGN_OR_RETURN(cel::Value converted_value, + PyObjectToCelValue( + py_opt.RawValue(), element_type, + [context]() { + return absl::StrFormat("%s.value()", + context()); + }, + env, arena)); + if (converted_value.IsError()) { + return converted_value; + } + return cel::OptionalValue::Of(std::move(converted_value), arena); + } + return absl::UnimplementedError(absl::StrFormat( + "Unimplemented opaque type: %s", expected_type.GetName())); + } default: { return absl::UnimplementedError( absl::StrFormat("Unimplemented type: %d", expected_type.GetKind())); diff --git a/cel_expr_python/py_cel_value.h b/cel_expr_python/py_cel_value.h index f82d3cc..6bdbc35 100644 --- a/cel_expr_python/py_cel_value.h +++ b/cel_expr_python/py_cel_value.h @@ -146,11 +146,13 @@ class PyCelMapItemAccessor : public PyCelValue { bool resolved_ ABSL_GUARDED_BY(mutex_) = false; }; +// This should be called with the GIL held. PyObject* CelValueToPyObject(const cel::Value& cel_value, const std::shared_ptr& env, const std::shared_ptr& arena, bool plain_value); +// This should be called with the GIL held. absl::StatusOr PyObjectToCelValue( PyObject* py_object, const PyCelType& expected_type, absl::FunctionRef context, diff --git a/cel_expr_python/py_message_factory.cc b/cel_expr_python/py_message_factory.cc index 080b33b..b2d96f0 100644 --- a/cel_expr_python/py_message_factory.cc +++ b/cel_expr_python/py_message_factory.cc @@ -43,11 +43,11 @@ PyMessageFactory::PyMessageFactory(PyObject* descriptor_pool) { ABSL_LOG(FATAL) << "Cannot load module 'google.protobuf.message_factory'"; } py_func_GetMessageClass_ = PyObject_GetAttrString(pModule, "GetMessageClass"); + Py_DECREF(pModule); if (!py_func_GetMessageClass_) { ABSL_LOG(FATAL) << "Cannot find function " "'google.protobuf.message_factory.GetMessageClass'"; } - Py_INCREF(py_func_GetMessageClass_); py_func_MergeFromString_ = PyUnicode_FromString("MergeFromString"); } @@ -146,12 +146,12 @@ PyObject* PyMessageFactory::FromString(const std::string& message_type, PyObject* ret = PyObject_CallMethodObjArgs(message, py_func_MergeFromString_, serialized_proto_py, nullptr); + Py_DECREF(serialized_proto_py); if (!ret) { Py_DECREF(message); - Py_DECREF(serialized_proto_py); return nullptr; } - Py_DECREF(serialized_proto_py); + Py_DECREF(ret); return message; } diff --git a/conformance/policy/BUILD b/conformance/policy/BUILD new file mode 100644 index 0000000..47f845a --- /dev/null +++ b/conformance/policy/BUILD @@ -0,0 +1,37 @@ +load("@rules_python//python:py_test.bzl", "py_test") + +package(default_visibility = ["//visibility:private"]) + +py_test( + name = "policy_conformance_test", + srcs = ["policy_conformance_test.py"], + data = [ + "//cel_expr_python:cel", + "//cel_expr_python/ext:ext_bindings", + "//cel_expr_python/ext:ext_optional", + "//cel_expr_python/policy:cel_policy", + "@cel_policy//conformance:testdata", + ], + env = { + "TESTDATA_DIR": "+_repo_rules+cel_policy/conformance/testdata/", + "TEST_FILES": "$(rlocationpaths @cel_policy//conformance:testdata)", + }, + deps = [ + "//testing:proto3_test_all_types_py_pb2", + "//testing:suite_py_pb2", + "@com_google_absl_py//absl/testing:absltest", + "@com_google_cel_spec//proto/cel/expr:checked_py_pb2", + "@com_google_cel_spec//proto/cel/expr:value_py_pb2", + "@com_google_protobuf//:protobuf_python", + "@pyyaml//:yaml", + "@rules_python//python/runfiles", + ] + select({ + "@platforms//os:windows": [], + "//conditions:default": [ + "//cel_expr_python:cel", + "//cel_expr_python/ext:ext_bindings", + "//cel_expr_python/ext:ext_optional", + "//cel_expr_python/policy:cel_policy", + ], + }), +) diff --git a/conformance/policy/policy_conformance_test.py b/conformance/policy/policy_conformance_test.py new file mode 100644 index 0000000..1b0af57 --- /dev/null +++ b/conformance/policy/policy_conformance_test.py @@ -0,0 +1,454 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Conformance tests for cel-python policy.""" + +import os +from typing import Any +import unittest + +from google.protobuf import descriptor_pool +from google.protobuf import message_factory +from google.protobuf import text_format +import yaml + +from cel.expr import value_pb2 as value_pb +from absl.testing import absltest +from python.runfiles import runfiles +from cel_expr_python import cel +from cel_expr_python.ext import ext_bindings +from cel_expr_python.ext import ext_optional +from cel_expr_python.policy import cel_policy +from cel.expr.conformance.proto3 import test_all_types_pb2 as test_all_types_proto3 # pylint: disable=unused-import +from cel.expr.conformance.test import suite_pb2 as suite_pb + + +class PolicyConformanceTestSuite(unittest.TestSuite): + SKIP_TESTS = [ + # TODO(b/506179116): Fix these. + # Need to add k8s custom yaml parser and mock runtime. + "k8s", + # Expects Java reverse-order type mismatch error message (matches C++ + # skip in cel-cpp conformance/policy/BUILD). + "compile_errors/aggregate_heterogeneous_outputs", + # Expects "rule creates unreachable outputs" on the exhaustive subrule, + # whereas C++ reports "match creates unreachable outputs" on the + # subsequent unreachable match (matches C++ skip in cel-cpp + # conformance/policy/BUILD). + "compile_errors/unreachable_under_unconditional_aggregate_subrule", + ] + + def __init__(self): + super().__init__(self) + self._load_tests() + + def _load_tests(self): + r = runfiles.Create() + env_test_files = os.environ.get("TEST_FILES") + if not env_test_files: + raise ValueError("TEST_FILES environment variable not set") + + testdata_dir = os.environ.get("TESTDATA_DIR") + if not testdata_dir: + raise ValueError("TESTDATA_DIR environment variable not set") + + subdirs = [] + for rel_path in env_test_files.split(testdata_dir): + rel_path = rel_path.strip() + if rel_path and rel_path.endswith("policy.yaml"): + f = testdata_dir + rel_path + location = r.Rlocation(f) + if not location: + raise ValueError(f"Could not resolve rlocation: {f}") + subdirs.append(os.path.dirname(location)) + + subdirs.sort() + + if not subdirs: + return + + testdata_path = subdirs[0] + while not testdata_path.endswith("testdata"): + parent = os.path.dirname(testdata_path) + if parent == testdata_path: + raise ValueError("Could not resolve testdata directory") + testdata_path = parent + + for root in subdirs: + rel_path = os.path.relpath(root, testdata_path) + suite_name = rel_path.replace("\\", "/") + + should_skip_suite = any( + suite_name == pattern or suite_name.startswith(pattern + "/") + for pattern in self.SKIP_TESTS + ) + if should_skip_suite: + print(f"[ SKIPPED SUITE ] {suite_name}") + continue + + policy_path = os.path.join(root, "policy.yaml") + with open(policy_path, "rt") as f: + policy_content = f.read() + + config_path = os.path.join(root, "config.yaml") + config_content = "" + context_msg_type_name = None + if os.path.exists(config_path): + with open(config_path, "rt") as f: + config_content = f.read() + env_cfg = cel.NewEnvConfigFromYaml(config_content) + context_msg_type_name = ( + env_cfg.context_type if env_cfg.context_type else None + ) + + tests_path = os.path.join(root, "tests.yaml") + if os.path.exists(tests_path): + with open(tests_path, "rt") as f: + self._load_yaml_tests( + suite_name, + f.read(), + policy_content, + config_content, + context_msg_type_name, + ) + else: + tests_path = os.path.join(root, "tests.textproto") + if os.path.exists(tests_path): + with open(tests_path, "rt") as f: + self._load_textproto_tests( + suite_name, + f.read(), + policy_content, + config_content, + context_msg_type_name, + ) + + def _load_yaml_tests( + self, + suite_name: str, + tests_content: str, + policy_content: str, + config_content: str, + context_msg_type_name: str | None, + ): + test_suite = yaml.safe_load(tests_content) + sections = test_suite.get("sections", test_suite.get("section", [])) + for section in sections: + section_name = section.get("name", "") + tests = section.get("tests", section.get("test", [])) + for test in tests: + test_name = test.get("name", "") + full_test_name = f"{suite_name}/{section_name}/{test_name}" + + context_expr = test.get("context_expr") + + inputs = {} + for var_name, val_spec in test.get("input", {}).items(): + if isinstance(val_spec, dict) and "expr" in val_spec: + inputs[var_name] = ("expr", val_spec["expr"]) + elif isinstance(val_spec, dict) and "value" in val_spec: + inputs[var_name] = ("value", val_spec["value"]) + else: + inputs[var_name] = ("value", val_spec) + + output_spec = test.get("output") + if isinstance(output_spec, dict): + if "expr" in output_spec: + expected = ("expr", output_spec["expr"]) + elif "value" in output_spec: + expected = ("value", output_spec["value"]) + elif "error" in output_spec: + expected = ("errors", [output_spec["error"]]) + elif "error_set" in output_spec: + expected = ("errors", output_spec["error_set"]) + elif "unknown" in output_spec: + expected = ("unknown", output_spec["unknown"]) + else: + expected = ("value", output_spec) + else: + expected = ("value", output_spec) + + test_spec = { + "full_test_name": full_test_name, + "context_expr": context_expr, + "inputs": inputs, + "expected": expected, + } + self.addTest( + PolicyConformanceTest( + full_test_name, + suite_name, + policy_content, + config_content, + context_msg_type_name, + test_spec, + ) + ) + + def _load_textproto_tests( + self, + suite_name: str, + tests_content: str, + policy_content: str, + config_content: str, + context_msg_type_name: str | None, + ): + test_suite = suite_pb.TestSuite() + text_format.Parse(tests_content, test_suite) + for section in test_suite.sections: + section_name = section.name + for test in section.tests: + test_name = test.name + full_test_name = f"{suite_name}/{section_name}/{test_name}" + + context_expr = None + if test.HasField("input_context"): + if test.input_context.HasField("context_expr"): + context_expr = test.input_context.context_expr + + inputs = {} + for var_name, input_val in test.input.items(): + if input_val.HasField("expr"): + inputs[var_name] = ("expr", input_val.expr) + elif input_val.HasField("value"): + inputs[var_name] = ("proto_value", input_val.value) + + if test.HasField("output"): + out_proto = test.output + if out_proto.HasField("result_expr"): + expected = ("expr", out_proto.result_expr) + elif out_proto.HasField("result_value"): + expected = ("proto_value", out_proto.result_value) + elif out_proto.HasField("eval_error"): + expected = ( + "errors", + [err.message for err in out_proto.eval_error.errors], + ) + elif out_proto.HasField("unknown"): + expected = ("unknown", list(out_proto.unknown.exprs)) + else: + expected = ("value", True) + else: + expected = ("value", True) + + test_spec = { + "full_test_name": full_test_name, + "context_expr": context_expr, + "inputs": inputs, + "expected": expected, + } + self.addTest( + PolicyConformanceTest( + full_test_name, + suite_name, + policy_content, + config_content, + context_msg_type_name, + test_spec, + ) + ) + + +class PolicyConformanceTest(absltest.TestCase): + + def __init__( + self, + test_name: str, + suite_name: str, + policy_content: str, + config_content: str, + _context_msg_type_name: str | None, + test_spec: dict[str, Any], + ): + self.descriptor_pool = descriptor_pool.Default() + + # CEL Environment for evaluating input expressions and expected results. + # Note: this is different from the CEL environment used for compiling the + # policy (i.e. `env` below). + self.test_env = cel.NewEnv( + descriptor_pool=self.descriptor_pool, + variables={"val": cel.Type.DYN}, + extensions=[ + ext_optional.ExtOptional(), + ], + ) + + def _function(): + self._run_policy_test( + suite_name, policy_content, config_content, test_spec + ) + + setattr(self, test_name, _function) + super().__init__(test_name) + + def _convert_value(self, value: value_pb.Value) -> Any: + if value.HasField("null_value"): + return None + elif value.HasField("bool_value"): + return value.bool_value + elif value.HasField("int64_value"): + return value.int64_value + elif value.HasField("uint64_value"): + return value.uint64_value + elif value.HasField("double_value"): + return value.double_value + elif value.HasField("string_value"): + return value.string_value + elif value.HasField("bytes_value"): + return value.bytes_value + elif value.HasField("object_value"): + desc = self.descriptor_pool.FindMessageTypeByName( + value.object_value.TypeName() + ) + proto = message_factory.GetMessageClass(desc)() + value.object_value.Unpack(proto) + return proto + elif value.HasField("list_value"): + return [self._convert_value(v) for v in value.list_value.values] + elif value.HasField("map_value"): + return { + self._convert_value(e.key): self._convert_value(e.value) + for e in value.map_value.entries + } + elif value.HasField("enum_value"): + return value.enum_value + else: + self.fail("Unsupported value type: " + str(value)) + + def _run_policy_test( + self, + suite_name: str, + policy_content: str, + config_content: str, + test_spec: dict[str, Any], + ): + parse_result = cel_policy.parse_yaml_cel_policy(policy_content) + if not parse_result.is_valid: + self.fail( + "Failed to parse policy.yaml in" + f" {suite_name}\nIssues:\n{parse_result.formatted_issues()}" + ) + policy = parse_result.policy() + + policy_env = cel.NewEnv( + descriptor_pool=self.descriptor_pool, + config=cel.NewEnvConfigFromYaml(config_content), + extensions=[ + ext_bindings.ExtBindings(), + ext_optional.ExtOptional(), + ], + function_impls={ + "locationCode_string": location_code_impl, + "hasCreditCard": has_credit_card_impl, + "hasEmailOrPhone": has_email_or_phone_impl, + }, + ) + + compile_result = cel_policy.compile(policy_env, policy) + + expected_type, expected_data = test_spec["expected"] + + if expected_type == "errors": + self.assertFalse( + compile_result.is_valid, + f"Expected compilation to fail in {test_spec['full_test_name']}", + ) + err_msg = compile_result.formatted_issues() + for expected_err in expected_data: + self.assertIn( + expected_err, + err_msg, + "Did not find expected compile time error in" + f" {test_spec['full_test_name']}", + ) + return + + self.assertTrue( + compile_result.is_valid, + f"Compilation has validation errors in {test_spec['full_test_name']}:" + f" {compile_result.formatted_issues()}", + ) + program = compile_result.expression() + + act_data = {} + if test_spec["context_expr"]: + context_res = self.test_env.compile(test_spec["context_expr"]).eval() + proto_obj = context_res.plain_value() + if hasattr(proto_obj, "DESCRIPTOR"): + for field in proto_obj.DESCRIPTOR.fields: + val = getattr(proto_obj, field.name) + if field.is_repeated: + if field.message_type and field.message_type.GetOptions().map_entry: + val = dict(val) + else: + val = list(val) + act_data[field.name] = val + else: + for var_name, (in_type, in_val) in test_spec["inputs"].items(): + if in_type == "expr": + res = self.test_env.compile(in_val).eval() + act_data[var_name] = res.plain_value() + elif in_type == "proto_value": + act_data[var_name] = self._convert_value(in_val) + else: + act_data[var_name] = in_val + + activation = policy_env.Activation(act_data) + actual_res = program.eval(activation) + actual_repr = repr(actual_res) + + if expected_type == "expr": + expected_res = self.test_env.compile(expected_data).eval() + expected_repr = repr(expected_res) + elif expected_type == "proto_value": + plain_val = self._convert_value(expected_data) + expected_res = self.test_env.compile("val").eval( + self.test_env.Activation({"val": plain_val}) + ) + expected_repr = repr(expected_res) + else: + expected_res = self.test_env.compile("val").eval( + self.test_env.Activation({"val": expected_data}) + ) + expected_repr = repr(expected_res) + + if actual_repr.startswith("optional.of(") and actual_repr.endswith(")"): + if not expected_repr.startswith("optional."): + actual_repr = actual_repr[len("optional.of(") : -1] + + self.assertEqual( + actual_repr, + expected_repr, + f"Test failed: {test_spec['full_test_name']}", + ) + + +def location_code_impl(ip: str) -> str: + if ip == "10.0.0.1": + return "us" + if ip == "10.0.0.2": + return "de" + return "ir" + + +def has_credit_card_impl(arg: Any) -> bool: + return isinstance(arg, dict) and "cc" in arg + + +def has_email_or_phone_impl(arg: Any) -> bool: + return isinstance(arg, dict) and ("email" in arg or "phone" in arg) + + +if __name__ == "__main__": + absltest.main(defaultTest="PolicyConformanceTestSuite") diff --git a/release/build_wheel.sh b/release/build_wheel.sh index eeeef3f..6b45fb8 100755 --- a/release/build_wheel.sh +++ b/release/build_wheel.sh @@ -69,7 +69,7 @@ if [[ "$OSTYPE" == "darwin"* ]]; then else cp -r "${SRC_DIR}/." . fi -rm -rf cel_expr_python/*_test.py +rm -rf cel_expr_python/*_test.py cel_expr_python/leak_check.py PYTHON_BIN="python" if command -v python3 &> /dev/null; then diff --git a/release/cel_basic_test.py b/release/cel_basic_test.py index 7295243..d656641 100644 --- a/release/cel_basic_test.py +++ b/release/cel_basic_test.py @@ -77,6 +77,14 @@ def test_standard_extensions(self): # Test optional expr = env.compile("optional.of('test').orValue('fallback')") self.assertEqual(expr.eval().value(), "test") + self.assertEqual( + env.compile("optional.of('test')").eval().plain_value(), + cel.Optional.of("test"), + ) + self.assertEqual( + env.compile("optional.none()").eval().plain_value(), + cel.Optional.none(), + ) # Test strings expr = env.compile("strings.quote('abc')") @@ -192,6 +200,7 @@ def test_wheel_package_contents(self): ("cel_expr_python/", "cel_expr_python-", "cel_expr_python.libs/") ) or f.endswith("_test.py") + or f == "cel_expr_python/leak_check.py" or "/testdata/" in f ] self.assertEqual( diff --git a/release/kokoro/release_linux.sh b/release/kokoro/release_linux.sh index 08bfe0c..3ff4c95 100755 --- a/release/kokoro/release_linux.sh +++ b/release/kokoro/release_linux.sh @@ -243,7 +243,7 @@ pushd "${TMP_DIR}" cp -r "${SRC_DIR}"/{*,.*} . 2>/dev/null || true cp "${SRC_DIR}"/release/*.py "${SRC_DIR}"/release/*.toml . 2>/dev/null || true -rm -rf cel_expr_python/*_test.py +rm -rf cel_expr_python/*_test.py cel_expr_python/leak_check.py echo "Downloading bazelisk on host..." curl -LO https://github.com/bazelbuild/bazelisk/releases/download/v1.19.0/bazelisk-linux-amd64 diff --git a/release/kokoro/release_macos.sh b/release/kokoro/release_macos.sh index e36b555..52c1dd3 100644 --- a/release/kokoro/release_macos.sh +++ b/release/kokoro/release_macos.sh @@ -58,7 +58,7 @@ pushd "${TMP_DIR}" cp -r "${SRC_DIR}"/{*,.*} . 2>/dev/null || true cp "${SRC_DIR}"/release/*.py "${SRC_DIR}"/release/*.toml . 2>/dev/null || true -rm -rf cel_expr_python/*_test.py +rm -rf cel_expr_python/*_test.py cel_expr_python/leak_check.py # Check if pyproject.toml exists before running sed if [ -f pyproject.toml ]; then diff --git a/release/kokoro/release_windows.bat b/release/kokoro/release_windows.bat index 2931936..f0d7a37 100644 --- a/release/kokoro/release_windows.bat +++ b/release/kokoro/release_windows.bat @@ -131,6 +131,9 @@ if !ERRORLEVEL! NEQ 0 ( if exist "cel_expr_python\*_test.py" ( del /Q "cel_expr_python\*_test.py" ) +if exist "cel_expr_python\leak_check.py" ( + del /Q "cel_expr_python\leak_check.py" +) :: Substitute $VERSION in pyproject.toml with the value of VERSION. !PYTHON_EXE! -c "import sys; content = open('pyproject.toml').read(); open('pyproject.toml', 'w').write(content.replace('$VERSION', sys.argv[1]))" "%VERSION%" diff --git a/release/pyproject.toml b/release/pyproject.toml index 3b54d9e..f704343 100644 --- a/release/pyproject.toml +++ b/release/pyproject.toml @@ -42,7 +42,7 @@ exclude = ["*testdata*"] "*" = ["*.pyi"] [tool.setuptools.exclude-package-data] -"*" = ["*_test.py"] +"*" = ["*_test.py", "leak_check.py"] [tool.cibuildwheel] build = "cp310-* cp311-* cp312-* cp313-* cp314-*" diff --git a/testing/BUILD.bazel b/testing/BUILD.bazel index 28bf82a..64dba97 100644 --- a/testing/BUILD.bazel +++ b/testing/BUILD.bazel @@ -24,3 +24,9 @@ py_proto_library( visibility = ["//visibility:public"], deps = ["@cel_conformance//proto/cel/expr/conformance/test:simple_proto"], ) + +py_proto_library( + name = "suite_py_pb2", + visibility = ["//visibility:public"], + deps = ["@cel_conformance//proto/cel/expr/conformance/test:suite_proto"], +)