from __future__ import annotations
import abc
import functools
import numbers
from collections.abc import Hashable, Mapping
from typing import Any, NoReturn, Self
import numpy as np
from mpi4py import MPI
import pyop3.axis_tree
import pyop3.collections
import pyop3.exceptions
import pyop3.index_tree
import pyop3.record
from pyop3 import utils
from pyop3.axis_tree import UNIT_AXIS_TREE, AxisTree
from pyop3.dtypes import IntType, RealType
from pyop3.node import Node, Terminal
[docs]
class Expression(Node, abc.ABC):
# {{{ abstract methods
@property
def local_max(self) -> numbers.Number:
raise NotImplementedError
@property
def local_min(self) -> numbers.Number:
raise NotImplementedError
@property
def dtype(self):
raise NotImplementedError(f"dtype not implemented for {utils.pretty_type(self)}")
@property
@abc.abstractmethod
def _full_str(self) -> str:
pass
# }}}
def __str__(self) -> str:
return self._full_str
# {{{ arithmetic
# TODO: all of these shortcuts do not work with record_new, need to overwrite __new__
# for universal behaviour
def __add__(self, other: ExpressionT, /) -> Expression:
if other == 0:
return self
else:
return Add(self, other)
def __radd__(self, other: ExpressionT, /) -> Expression:
if other == 0:
return self
else:
return Add(other, self)
def __sub__(self, other) -> Sub | Self:
if other == 0:
return self
else:
return Sub(self, other)
def __rsub__(self, other) -> Sub | Self:
if other == 0:
return self
else:
return Sub(other, self)
def __mul__(self, other) -> Mul | Self:
if other == 0:
return 0
elif other == 1:
return self
else:
return Mul(self, other)
def __rmul__(self, other) -> Mul | Self:
if other == 0:
return 0
elif other == 1:
return self
else:
return Mul(other, self)
def __truediv__(self, other) -> Div | Self:
if other == 1:
return self
else:
return Div(self, other)
def __rtruediv__(self, other) -> Div | Self:
return Div(other, self)
def __floordiv__(self, other) -> FloorDiv | Self:
if not isinstance(other, numbers.Integral):
return NotImplemented
if other == 1:
return self
else:
return FloorDiv(self, other)
def __mod__(self, other) -> Modulo | Self:
# TODO: raise nice exception
assert isinstance(other, numbers.Number)
if other == 1:
return self
else:
return Modulo(self, other)
def __neg__(self) -> Neg:
if isinstance(self, Neg):
# Neg(Neg(obj)) == obj
return self.operand
else:
return Neg(self)
def __pow__(self, other, /) -> Pow:
return Pow(self, other)
def __abs__(self) -> Abs:
if isinstance(self, Abs):
# Abs(Abs(obj)) == Abs(obj)
return self.operand
else:
return Abs(self)
def __lt__(self, other):
return LessThan(self, other)
def __gt__(self, other):
return GreaterThan(self, other)
def __le__(self, other):
return LessThanOrEqual(self, other)
def __ge__(self, other):
return GreaterThanOrEqual(self, other)
def __or__(self, other) -> Or | bool:
return self._maybe_eager_or(self, other)
def __ror__(self, other) -> Or | bool:
return self._maybe_eager_or(other, self)
# }}}
@classmethod
def _maybe_eager_or(cls, a, b) -> Or | Expression | bool:
from pyop3 import evaluate
try:
a_result = evaluate(a)
except pyop3.exceptions.MissingVariableException:
a_result = None
try:
b_result = evaluate(b)
except pyop3.exceptions.MissingVariableException:
b_result = None
if a_result or b_result:
return True
elif a_result is False:
if b_result is False:
return False
else:
assert b_result is None
return b
else:
assert a_result is None
if b_result is False:
return a
else:
assert b_result is None
return Or(a, b)
[docs]
class Operator(Expression, metaclass=abc.ABCMeta):
@property
@abc.abstractmethod
def operands(self) -> tuple[ExpressionT, ...]:
pass
[docs]
@abc.abstractmethod
def with_operands(self, operands) -> Self:
pass
[docs]
@pyop3.record.frozenrecord()
class UnaryOperator(Operator, metaclass=abc.ABCMeta):
# {{{ instance attrs
a: ExpressionT
[docs]
def collect_buffers(self, visitor):
return visitor(self.a)
[docs]
def get_disk_cache_key(self, visitor):
return (type(self), visitor(self.a))
get_instruction_executor_cache_key = get_disk_cache_key
# }}}
# {{{ interface impls
@property
def operands(self) -> tuple[ExpressionT]:
return (self.a,)
[docs]
def with_operands(self, operands):
a, = operands
return self.record_new(a=a)
child_attrs = ("a",)
@property
def dtype(self) -> str:
a_dtype = _get_dtype(self.a)
return a_dtype
@property
def _full_str(self) -> str:
return f"{self.symbol}{as_str(self.a)}"
# }}}
# {{{ abstract methods
@property
@abc.abstractmethod
def symbol(self) -> str:
pass
# }}}
@property
def operand(self):
return utils.just_one(self.operands)
[docs]
class Neg(UnaryOperator):
@property
def symbol(self) -> str:
return "-"
@property
def local_max(self) -> numbers.Number:
return -self.a.local_min
@property
def local_min(self) -> numbers.Number:
return -self.a.local_max
[docs]
class Abs(UnaryOperator):
@property
def symbol(self) -> str:
return "abs."
@property
def local_max(self) -> numbers.Number:
raise NotImplementedError
@property
def local_min(self) -> numbers.Number:
raise NotImplementedError
[docs]
@pyop3.record.frozenrecord()
class BinaryOperator(Operator, metaclass=abc.ABCMeta):
# {{{ instance attrs
a: ExpressionT
b: ExpressionT
[docs]
def collect_buffers(self, visitor):
return visitor(self.a) | visitor(self.b)
[docs]
def get_disk_cache_key(self, visitor):
return (type(self), visitor(self.a), visitor(self.b))
get_instruction_executor_cache_key = get_disk_cache_key
# }}}
# {{{ interface impls
child_attrs = ("a", "b")
@property
def operands(self) -> tuple[ExpressionT, ExpressionT]:
return (self.a, self.b)
[docs]
def with_operands(self, operands):
a, b = operands
return self.record_new(a=a, b=b)
@property
def dtype(self) -> np.dtype:
a_dtype = _get_dtype(self.a)
b_dtype = _get_dtype(self.b)
return np.result_type(a_dtype, b_dtype)
# }}}
# {{{ abstract methods
@property
@abc.abstractmethod
def _symbol(self) -> str:
pass
# }}}
@property
def _full_str(self) -> str:
# Always use brackets to avoid having to deal with operator precedence rules
return f"({as_str(self.a)} {self._symbol} {as_str(self.b)})"
[docs]
class Add(BinaryOperator):
# {{{ interface impls
@property
def _symbol(self) -> str:
return "+"
@property
def local_max(self) -> numbers.Number:
from pyop3.expr.visitors import get_local_max
return get_local_max(self.a) + get_local_max(self.b)
@property
def local_min(self) -> numbers.Number:
from pyop3.expr.visitors import get_local_min
return get_local_min(self.a) + get_local_min(self.b)
# }}}
[docs]
class Sub(BinaryOperator):
# {{{ interface impls
@property
def _symbol(self) -> str:
return "-"
@property
def local_max(self) -> numbers.Number:
from pyop3.expr.visitors import get_local_max, get_local_min
return get_local_max(self.a) - get_local_min(self.b)
@property
def local_min(self) -> numbers.Number:
from pyop3.expr.visitors import get_local_max, get_local_min
return get_local_min(self.a) - get_local_max(self.b)
# }}}
[docs]
class Mul(BinaryOperator):
def __new__(cls, a, b):
if a == 0 or b == 0:
return 0
elif a == 1:
return b
elif b == 1:
return a
else:
self = object.__new__(cls)
object.__setattr__(self, "a", a)
object.__setattr__(self, "b", b)
return self
# {{{ interface impls
@property
def _symbol(self) -> str:
return "*"
@property
def local_max(self) -> numbers.Number:
from pyop3.expr.visitors import get_local_max
return get_local_max(self.a) * get_local_max(self.b)
@property
def local_min(self) -> numbers.Number:
from pyop3.expr.visitors import get_local_min
return get_local_min(self.a) * get_local_min(self.b)
# }}}
[docs]
class Div(BinaryOperator):
@property
def _symbol(self) -> str:
return "/"
[docs]
class FloorDiv(BinaryOperator):
@property
def _symbol(self) -> str:
return "//"
[docs]
class Modulo(BinaryOperator):
@property
def _symbol(self) -> str:
return "%"
[docs]
class Pow(BinaryOperator):
@property
def _symbol(self) -> str:
return "**"
@property
def local_max(self) -> numbers.Number:
raise NotImplementedError
@property
def local_min(self) -> numbers.Number:
raise NotImplementedError
[docs]
class Comparison(BinaryOperator, metaclass=abc.ABCMeta):
# {{{ interface impls
@property
def local_max(self) -> numbers.Number:
raise TypeError("not sure that this makes sense")
@property
def local_min(self) -> numbers.Number:
raise TypeError("not sure that this makes sense")
# }}}
[docs]
class LessThan(Comparison):
@property
def _symbol(self) -> str:
return "<"
[docs]
class GreaterThan(Comparison):
@property
def _symbol(self) -> str:
return ">"
[docs]
class LessThanOrEqual(Comparison):
@property
def _symbol(self) -> str:
return "<="
[docs]
class GreaterThanOrEqual(Comparison):
@property
def _symbol(self) -> str:
return ">="
[docs]
class Or(Comparison):
@property
def _symbol(self) -> str:
return "|"
[docs]
@pyop3.record.frozenrecord()
class TernaryOperator(Operator, metaclass=abc.ABCMeta):
# {{{ instance attrs
a: ExpressionT
b: ExpressionT
c: ExpressionT
[docs]
def collect_buffers(self, visitor):
return visitor(self.a) | visitor(self.b) | visitor(self.c)
[docs]
def get_disk_cache_key(self, visitor):
return (type(self), visitor(self.a), visitor(self.b), visitor(self.c))
get_instruction_executor_cache_key = get_disk_cache_key
# }}}
# {{{ interface impls
child_attrs = ("a", "b", "c")
@property
def operands(self) -> tuple[ExpressionT, ExpressionT, ExpressionT]:
return (self.a, self.b, self.c)
[docs]
def with_operands(self, operands):
a, b, c = operands
return self.record_new(a=a, b=b, c=c)
@property
def dtype(self) -> np.dtype:
a_dtype = _get_dtype(self.a)
b_dtype = _get_dtype(self.b)
c_dtype = _get_dtype(self.c)
return np.result_type(a_dtype, b_dtype, c_dtype)
# }}}
[docs]
@pyop3.record.frozenrecord()
class Conditional(TernaryOperator):
# {{{ interface impls
@property
def _full_str(self) -> str:
return f"{as_str(self.predicate)} ? {as_str(self.if_true)} : {as_str(self.if_false)}"
# }}}
@property
def predicate(self) -> ExpressionT:
return self.a
@property
def if_true(self) -> ExpressionT:
return self.b
@property
def if_false(self) -> ExpressionT:
return self.c
@property
def local_max(self) -> numbers.Number:
from pyop3.expr.visitors import get_local_max
return max(*map(get_local_max, [self.if_true, self.if_false]))
@property
def local_min(self) -> numbers.Number:
raise TypeError("not sure that this makes sense")
[docs]
def conditional(predicate, if_true, if_false):
from pyop3 import evaluate
if if_true == if_false:
return if_true
try:
predicate = evaluate(predicate)
except pyop3.exceptions.MissingVariableException:
return Conditional(predicate, if_true, if_false)
else:
assert isinstance(predicate, bool)
return if_true if predicate else if_false
[docs]
class TerminalExpression(Expression, Terminal, abc.ABC):
child_attrs = ()
[docs]
@pyop3.record.frozenrecord()
class NameVar(TerminalExpression):
"""Class for representing arbitrary variables.
They are useful for evaluating arbitrary symbolic expressions where something
like an `AxisVar` would be inappropriate (due to there not being an axis
available).
"""
name: Hashable
[docs]
def dtype(self) -> np.dtype:
raise TypeError("dtype of NameVars cannot be determined")
def _full_str(self) -> str:
return f"{utils.pretty_type(self)}('{self.name}')"
[docs]
class NamedTerminalExpression(TerminalExpression):
"""A terminal with a name.
This type is important because only named terminals can be replaced when
an operation is reused. For example we can only do the following:
.. code-block:: python
loop = op3.loop(p, kernel(dat1[p]))
loop(**{"dat": dat2}) # pass dat2 instead of dat1
if ``dat1`` is a named terminal.
"""
__abstract_record_attrs = ("name",)
[docs]
@pyop3.record.frozenrecord()
class AxisVar(TerminalExpression):
# {{{ instance attrs
axis: pyop3.axis_tree.Axis
[docs]
def collect_buffers(self, visitor):
# Axis vars are just pointers to some outer loop. Any internal
# buffers that we need will be referenced elsewhere.
return pyop3.collections.OrderedFrozenSet()
[docs]
def get_disk_cache_key(self, visitor) -> Hashable:
# Axis vars are just pointers to some outer loop. We don't need to
# recurse here if we've seen the axis before, just make sure that
# the labels match.
if visitor.recurse_vars and self.axis.label not in visitor.seen_axes:
return (type(self), visitor(self.axis))
else:
return (type(self), visitor.renamer.add_type(pyop3.axis_tree.Axis, self.axis.label))
get_instruction_executor_cache_key = get_disk_cache_key
def __record_post_init(self) -> None:
assert len(self.axis.components) == 1
assert self.axis.component.sf is None
# }}}
def __eq__(self, other, /) -> bool:
return type(other) is type(self) and other._hashkey == self._hashkey
def __hash__(self, /) -> int:
return hash(self._hashkey)
@property
def _hashkey(self, /) -> tuple:
"""Key used for hashing and equality checks."""
# Note that self.axis.label and self.axis.size are used here because
# those are the only attributes that we care about for code generation.
# AxisVars are generated in a number of places and we cannot guarantee
# agreement on everything.
return (type(self), self.axis.label, self.axis.size)
# {{{ interface impls
@property
def local_max(self) -> numbers.Number:
raise TypeError("not sure that this makes sense")
@property
def local_min(self) -> numbers.Number:
raise TypeError("not sure that this makes sense")
@property
def dtype(self) -> np.dtype:
return IntType
@property
def _full_str(self) -> str:
return f"i_{{{self.axis.label}}}"
# }}}
[docs]
@pyop3.record.frozenrecord()
class NaN(TerminalExpression):
# {{{ pyop3.obj.Object interface impls
[docs]
def get_disk_cache_key(self, visitor) -> Hashable:
return (type(self),)
get_instruction_executor_cache_key = get_disk_cache_key
[docs]
@classmethod
def get_custom_comm(cls) -> MPI.Comm:
return MPI.COMM_SELF
# }}}
# {{{ interface impls
@property
def local_max(self) -> NoReturn:
raise TypeError
@property
def local_min(self) -> NoReturn:
raise TypeError
@property
def dtype(self) -> NoReturn:
raise TypeError
_full_str = "NaN"
# }}}
[docs]
@pyop3.record.frozenrecord()
class LoopIndexVar(TerminalExpression):
# {{{ instance attrs
loop_index: pyop3.index_tree.LoopIndex
axis: pyop3.axis_tree.Axis
[docs]
def get_disk_cache_key(self, visitor) -> Hashable:
# Loop index vars are just pointers to some outer loop. We don't need
# to recurse here if we've seen the loop index before, just make sure
# that the labels match.
if visitor.recurse_vars and self.loop_index.label not in visitor.seen_loop_indices:
return (type(self), visitor(self.loop_index), visitor(self.axis))
else:
return (
type(self),
visitor.renamer.add_type(pyop3.index_tree.LoopIndex, self.loop_index.label),
visitor.renamer.add_type(pyop3.axis_tree.Axis, self.axis.label),
)
get_instruction_executor_cache_key = get_disk_cache_key
[docs]
def collect_buffers(self, visitor):
# Loop index vars are just pointers to some outer loop. Any internal
# buffers that we need will be referenced elsewhere.
return pyop3.collections.OrderedFrozenSet()
def __init__(self, loop_index, axis) -> None:
from pyop3 import LoopContextFreeLoopIndex
# we must be linear at this point
assert len(axis.components) == 1
assert isinstance(loop_index, LoopContextFreeLoopIndex)
assert axis.component.sf is None
object.__setattr__(self, "loop_index", loop_index)
object.__setattr__(self, "axis", axis)
# }}}
# {{{ interface impls
@property
def local_max(self) -> numbers.Number:
raise TypeError("not sure that this makes sense")
@property
def local_min(self) -> numbers.Number:
raise TypeError("not sure that this makes sense")
@property
def dtype(self) -> np.dtype:
return IntType
@property
def _full_str(self) -> str:
return f"L_{{{self.loop_index.id}, {self.axis.label}}}"
# }}}
def __repr__(self) -> str:
return f"{type(self).__name__}({self.loop_index!r}, {self.axis.label!r})"
ExpressionT = Expression | numbers.Number
[docs]
@functools.singledispatch
def as_str(expr):
return expr._full_str
@as_str.register(Expression)
def _(expr):
return expr._full_str
@as_str.register(numbers.Number)
@as_str.register(bool)
@as_str.register(np.bool)
def _(expr):
return str(expr)
@functools.singledispatch
def _get_dtype(expr: Any) -> np.dtype:
return None
@_get_dtype.register(np.number)
def _(expr) -> np.dtype:
raise NotImplementedError(f"Not implemented dtype for: {type(expr)}")
@_get_dtype.register(Expression)
def _(expr: Expression) -> np.dtype:
return expr.dtype
@_get_dtype.register(numbers.Real)
@_get_dtype.register(np.floating)
def _(expr) -> np.dtype:
return RealType
@_get_dtype.register(numbers.Integral)
@_get_dtype.register(np.integer)
def _(expr) -> np.dtype:
return IntType
[docs]
def get_loop_tree(expr) -> tuple[AxisTree, Mapping[LoopIndexVar, AxisVar]]:
from pyop3.expr.visitors import collect_loop_index_vars
axes = []
loop_var_replace_map = {}
for loop_var in collect_loop_index_vars(expr):
axis = loop_var.axis
new_axis_label = f"{axis.label}_{loop_var.loop_index.id}"
new_axis = axis.record_new(label=new_axis_label)
axes.append(new_axis)
loop_var_replace_map[loop_var] = AxisVar(new_axis)
return (AxisTree.from_iterable(axes), loop_var_replace_map)
[docs]
def loopified_shape(expr: Expression) -> tuple[AxisTree, Mapping[LoopIndexVar, AxisVar]]:
import pyop3.visitors
from pyop3.expr.visitors import get_shape
loop_tree, loop_var_replace_map = get_loop_tree(expr)
# assume single tree for now
shape = utils.just_one(get_shape(expr))
if shape is UNIT_AXIS_TREE:
if loop_tree:
axis_tree = loop_tree
else:
axis_tree = UNIT_AXIS_TREE
else:
# Replace any references to the loop indices
new_node_map = {}
for path, axis in shape.node_map.items():
if axis is None:
new_node_map[path] = None
continue
new_components = []
for component in axis.components:
new_regions = []
for region in component.regions:
new_size = pyop3.visitors.replace(region.size, loop_var_replace_map)
new_regions.append(region.record_new(size=new_size))
new_regions = tuple(new_regions)
new_components.append(component.record_new(regions=new_regions))
new_node_map[path] = axis.record_new(components=tuple(new_components))
subtree = AxisTree(new_node_map)
axis_tree = loop_tree.add_subtree(loop_tree.leaf_path, subtree)
assert not axis_tree._all_region_labels
return axis_tree, loop_var_replace_map