Source code for dantro.data_ops.expr_ops

"""Implements data operations that work with expressions, e.g. lambda function
definitions or symbolic math"""

import logging
import math
import re
from typing import Callable, Tuple, Union

import numpy as np

from .._import_tools import LazyLoader

log = logging.getLogger(__name__)

xr = LazyLoader("xarray")
nx = LazyLoader("networkx")
pd = LazyLoader("pandas")
scipy = LazyLoader("scipy")

# -----------------------------------------------------------------------------

[docs]def expression( expr: str, *, symbols: dict = None, evaluate: bool = True, transformations: Tuple[Callable] = None, astype: Union[type, str] = float, ): """Parses and evaluates a symbolic math expression using SymPy. For parsing, uses sympy's :py:func:`sympy.parsing.sympy_parser.parse_expr`. The ``symbols`` are provided as ``local_dict``; the ``global_dict`` is not explicitly set and subsequently uses the sympy default value, containing all basic sympy symbols and notations. .. note:: The expression given here is not Python code, but symbolic math. You cannot call arbitrary functions, but only those that are imported by ``from sympy import *``. .. hint:: When using this expression as part of the :ref:`dag_framework`, it is attached to a so-called :ref:`syntax hook <dag_op_hooks_integration>` that makes it easier to specify the ``symbols`` parameter. See :ref:`here <dag_op_hook_expression>` for more information. .. warning:: While the expression is symbolic math, be aware that smypy by default interprets the ``^`` operator as XOR, not an exponentiation! For exponentiation, use the ``**`` operator or adjust the ``transformations`` argument as specified in the sympy documentation. .. warning:: The return object of this operation will *only* contain symbolic sympy objects if ``astype is None``. Otherwise, the type cast will evaluate all symbolic objects to the numerical equivalent specified by the given ``astype``. Args: expr (str): The expression to evaluate symbols (dict, optional): The symbols to use evaluate (bool, optional): Controls whether sympy evaluates ``expr``. This *may* lead to a fully evaluated result, but does not guarantee that no sympy objects are contained in the result. For ensuring a fully numerical result, see the ``astype`` argument. transformations (Tuple[Callable], optional): The ``transformations`` argument for sympy's :py:func:`sympy.parsing.sympy_parser.parse_expr`. By default, the sympy standard transformations are performed. astype (Union[type, str], optional): If given, performs a cast to this data type, fully evaluating all symbolic expressions. Default: Python :py:class:`float`. Raises: TypeError: Upon failing ``astype`` cast, e.g. due to free symbols remaining in the evaluated expression. ValueError: When parsing of ``expr`` failed. Returns: The result of the evaluated expression. """ from sympy.parsing.sympy_parser import parse_expr as _parse_expr from sympy.parsing.sympy_parser import standard_transformations as _std_trf log.remark("Evaluating symbolic expression: %s", expr) symbols = symbols if symbols else {} parse_kwargs = dict( evaluate=evaluate, transformations=( transformations if transformations is not None else _std_trf ), ) # Now, parse the expression try: res = _parse_expr(expr, local_dict=symbols, **parse_kwargs) except Exception as exc: raise ValueError( f"Failed parsing expression '{expr}'! Got a " f"{exc.__class__.__name__}: {exc}. Check that the expression can " "be evaluated with the available symbols " f"({', '.join(symbols) if symbols else 'none specified'}) " "and inspect the chained exceptions for more information. Parse " f"arguments were: {parse_kwargs}" ) from exc # Finished here if no type cast is desired if astype is None: return res # If full evaluation is desired, do so via a numpy type cast. This works on # all sympy objects, but _importantly_ also works on numpy arrays # containing sympy objects. dtype = np.dtype(astype) log.debug("Applying %s ...", dtype) try: return dtype.type(res) except Exception as exc: raise TypeError( f"Failed casting the result of expression '{expr}' from " f"{type(res)} to {dtype}! This can also be due to free symbols " "remaining in the evaluated expression. Either specify the free " f"symbols (got: {', '.join(symbols) if symbols else 'none'}) or " "deactivate casting by specifying None as ``dtype`` argument. " f"The expression evaluated to:\n\n {res}\n" ) from exc
[docs]def generate_lambda(expr: str) -> Callable: """Generates a lambda from a string. This is useful when working with callables in other operations. The ``expr`` argument needs to be a valid Python `lambda expression <>`_. Inside the lambda body, the following names are available for use: * A large part of the ``builtins`` module * Every name from the Python ``math`` module, e.g. ``sin``, ``cos``, … * These modules (and their long form): ``np``, ``xr``, ``scipy`` Internally, this uses ``eval`` but imposes the following restrictions: * The following strings may *not* appear in ``expr``: ``;``, ``__``. * There can be no nested ``lambda``, i.e. the only allowed lambda string is that in the beginning of ``expr``. * The dangerous parts from the ``builtins`` module are *not* available. Args: expr (str): The expression string to evaluate into a lambda. Returns: Callable: The generated Callable. Raises: SyntaxError: Upon failed evaluation of the given expression, invalid expression pattern, or disallowed strings in the lambda body. """ ALLOWED_BUILTINS = ( "abs", "all", "any", "callable", "chr", "divmod", "format", "hash", "hex", "id", "isinstance", "issubclass", "iter", "len", "max", "min", "next", "oct", "ord", "print", "repr", "round", "sorted", "sum", "None", "Ellipsis", "False", "True", "bool", "bytes", "bytearray", "complex", "dict", "enumerate", "filter", "float", "frozenset", "int", "list", "map", "range", "reversed", "set", "slice", "str", "tuple", "type", "zip", "open", ) DISALLOWED_STRINGS = ("lambda", ";", "__") LAMBDA_PATTERN = r"^\s*lambda\s[\w\s\,\*]+\:(.+)$" # arguments definitions : lambda body (capture group) # See also: # Check if the given expression matches this pattern pattern = re.compile(LAMBDA_PATTERN) match = pattern.match(expr) if match is None: raise SyntaxError( f"The given expression '{expr}' was not a valid lambda expression!" ) # Check if the lambda body contains disallowed strings lambda_body = match[1] if any([bad_str in lambda_body for bad_str in DISALLOWED_STRINGS]): raise SyntaxError( "Encountered one or more disallowed strings in the " f"body ('{lambda_body}') of the given lambda " f"expression ('{expr}'). Make sure none of the " "following strings appears there: " f"{DISALLOWED_STRINGS}" ) # Ok, sanitized enough now. Prepare the globals dict, restricting access to # a subset of builtins and only allowing commonly used math functionality _g = dict( __builtins__={ n: f for n, f in __builtins__.items() if n in ALLOWED_BUILTINS }, math=math, scipy=scipy, pandas=pd, numpy=np, xarray=xr, networkx=nx, np=np, xr=xr, nx=nx, pd=pd, **{n: f for n, f in math.__dict__.items() if not n.startswith("_")}, ) # Try evaluation now (with empty locals) try: f = eval(expr, _g, {}) except Exception as exc: raise SyntaxError( "Failed generating lambda object from expression " f"'{expr}'! Got a {exc.__class__.__name__}: {exc}" ) from exc return f