Source code for pyop3.visitors

from __future__ import annotations

import contextlib
import functools
import numbers
import types
from collections.abc import Hashable
from typing import Any

from immutabledict import immutabledict as idict

import pyop3.constants
import pyop3.node
import pyop3.obj
import pyop3.sf
import pyop3.visitors.base
import pyop3.visitors.identity
from pyop3 import utils
from pyop3.collections import OrderedFrozenSet

from pyop3.visitors.comm import common_comm, get_comm, single_comm  # noqa: F401
from pyop3.visitors.compress_indirections import materialize_indirections  # noqa: F401
from pyop3.visitors.concretize import concretize  # noqa: F401
from pyop3.visitors.relabel import Relabeler, relabel  # noqa: F401
from pyop3.visitors.replace import replace  # noqa: F401


[docs] class BufferCollector(pyop3.node.NodeCollector):
[docs] @functools.singledispatchmethod def process(self, obj: Any, /) -> OrderedFrozenSet: return super().process(obj)
@process.register(types.NoneType) @process.register(numbers.Number) def _(self, obj: Any, /) -> OrderedFrozenSet: return OrderedFrozenSet() @process.register def _(self, obj: pyop3.obj.Object, /) -> OrderedFrozenSet: return obj.collect_buffers(self)
# def collect_buffers(insn: pyop3.insn.Instruction) -> OrderedFrozenSet: # return BufferCollector.maybe_singleton(insn.comm)(insn)
[docs] def collect_buffers(obj) -> OrderedFrozenSet: return BufferCollector()(obj)
[docs] class CacheKeyGetter(pyop3.node.NodeVisitor): def __init__(self, recurse_vars: bool = True) -> None: self.recurse_vars = recurse_vars self.renamer = pyop3.visitors.base.Renamer() self.seen_loop_indices = [] self.seen_axes = [] super().__init__() # NOTE: copied from pyop3/visitors/relabel.py
[docs] def relabel_axis_tree_path(self, path): new_path = {} for axis, component in path.items(): new_axis = self.renamer.add_type(pyop3.axis_tree.Axis, axis) new_path[new_axis] = component return idict(new_path)
[docs] @contextlib.contextmanager def active_loop_index(self, label): self.seen_loop_indices.append(label) yield self.seen_loop_indices.pop(-1)
[docs] @contextlib.contextmanager def active_axis(self, label): self.seen_axes.append(label) yield self.seen_axes.pop(-1)
[docs] class DiskCacheKeyGetter(CacheKeyGetter): def __init__(self): super().__init__(recurse_vars=False)
[docs] @functools.singledispatchmethod def process(self, obj: Any) -> Hashable: return super().process(obj)
@process.register(types.NoneType) @process.register(numbers.Number) def _(self, obj: Any, /) -> Hashable: return obj @process.register def _(self, obj: pyop3.obj.Object, /) -> Hashable: return obj.get_disk_cache_key(self)
# TODO: This cache key is slightly too restrictive. For instance an axis tree and # indexed axis tree can be used identically in places (the output code is unchanged # and you'd get the same result) but currently these hash differently.
[docs] def get_disk_cache_key(obj: pyop3.obj.Object) -> Hashable: return DiskCacheKeyGetter()(obj)
[docs] class MemoryCacheKeyGetter(CacheKeyGetter): def __init__(self, *, identity_hash_outer_buffers: bool = True, **kwargs) -> None: self._identity_hash_buffers = identity_hash_outer_buffers super().__init__(**kwargs)
[docs] @contextlib.contextmanager def identity_hash_buffers(self): prev = self._identity_hash_buffers self._identity_hash_buffers = True yield self._identity_hash_buffers = prev
[docs] def get_cache_key(self, obj: pyop3.obj.Object, /, **kwargs): return (*super().get_cache_key(obj, **kwargs), self._identity_hash_buffers)
[docs] @functools.singledispatchmethod def process(self, obj: Any, /) -> Hashable: utils.raise_missing_dispatch_handler(obj)
@process.register(types.NoneType) @process.register(numbers.Number) def _(self, obj: Any, /) -> Hashable: return obj @process.register def _(self, obj: pyop3.obj.Object) -> Hashable: # TODO: rename this to get_memory_cache_key return obj.get_instruction_executor_cache_key(self)
[docs] def get_cache_key(obj: pyop3.obj.Object) -> Hashable: return MemoryCacheKeyGetter()(obj)
[docs] def get_instruction_executor_cache_key(obj: pyop3.obj.Object) -> Hashable: """ This cache key is different to, say, a disk cache key because it happens at the start of a calculation Also we only care about the top-level input buffers - buffers from things like indirection maps aren't considered replaceable because the idea is that we pass the input expression in here and get something back that we can reuse if we only change the top level buffers. e.g. dat1.assign(dat2) is the same as dat3.assign(dat4) if dat1/dat2 have the same axis trees as dat3/dat4. We can reuse the indirection maps and preprocessing optimisations etc and just change the buffers at the top-level. """ return MemoryCacheKeyGetter(identity_hash_outer_buffers=False, recurse_vars=False)(obj)