Source code for firedrake.ensemble.ensemble

from functools import wraps
import weakref
from contextlib import contextmanager
from types import SimpleNamespace

from firedrake.petsc import PETSc
from firedrake.function import Function
from firedrake.cofunction import Cofunction
from firedrake.exceptions import CommMismatchError
from pyop3.mpi import MPI, internal_comm


def _ensemble_mpi_dispatch(func):
    """
    This wrapper checks if any arg or kwarg of the wrapped
    ensemble method is a Function or Cofunction, and if so
    it calls the specialised Firedrake implementation.
    Otherwise the standard mpi4py implementation is called.
    """
    @wraps(func)
    def _mpi_dispatch(self, *args, **kwargs):
        if any(isinstance(arg, (Function, Cofunction))
               for arg in [*args, *kwargs.values()]):
            return func(self, *args, **kwargs)
        else:
            mpicall = getattr(self._ensemble_comm, func.__name__)
            return mpicall(*args, **kwargs)
    return _mpi_dispatch


[docs] class Ensemble: def __init__(self, comm: MPI.Comm, M: int, **kwargs): """ Create a set of space and ensemble subcommunicators. Wrapper methods around many MPI communication functions are provided for sending :class:`firedrake.function.Function` and :class:`.Cofunction` objects between spatial communicators. For non-Firedrake objects these wrappers will dispatch to the normal implementations on :class:`mpi4py.MPI.Comm`, which means that the same call site can be used for both Firedrake and non-Firedrake types. Parameters ---------- comm : The communicator to split. M : The size of the communicators used for spatial parallelism. Must be an integer divisor of the size of ``comm``. kwargs : Can include an ``ensemble_name`` string used as a communicator name prefix, for debugging. Raises ------ ValueError If ``M`` does not divide ``comm.size`` exactly. """ size = comm.size if (size // M)*M != size: raise ValueError("Invalid size of subcommunicators %d does not divide %d" % (M, size)) rank = comm.rank # Global comm self.global_comm = comm ensemble_name = kwargs.get("ensemble_name", "Ensemble") # User and internal communicator for spatial parallelism, contains a # contiguous chunk of M processes from `global_comm`. self.comm = self.global_comm.Split(color=(rank // M), key=rank) self.comm.name = f"{ensemble_name} spatial comm" weakref.finalize(self, self.comm.Free) # User and internal communicator for ensemble parallelism, contains all # processes in `global_comm` which have the same rank in `comm`. self.ensemble_comm = self.global_comm.Split(color=(rank % M), key=rank) self.ensemble_comm.name = f"{ensemble_name} ensemble comm" weakref.finalize(self, self.ensemble_comm.Free) # Keep a reference to the internal communicator because some methods return # non-blocking requests and we need to avoid cleaning up the communicator before # they complete. Note that this communicator should *never* be passed to PETSc, as # objects created with the communicator will never get cleaned up. self._ensemble_comm = internal_comm(self.ensemble_comm, self) if (self.comm.size != M) or (self.ensemble_comm.size != (size // M)): raise ValueError(f"{M=} does not exactly divide {comm.size=}") @property def ensemble_size(self) -> int: """The number of ensemble members. """ return self.ensemble_comm.size @property def ensemble_rank(self) -> int: """The rank of the local ensemble member. """ return self.ensemble_comm.rank def _check_function(self, f: Function | Cofunction, g: Function | Cofunction | None = None): """ Check if :class:`firedrake.function.Function` ``f`` (and possibly a second :class:`firedrake.function.Function` ``g``) is a valid argument for ensemble MPI routines Parameters ---------- f : The :class:`firedrake.function.Function` to check. g : Second :class:`firedrake.function.Function` to check. Raises ------ CommMismatchError If ``Function`` communicators mismatch each other or the ensemble spatial communicator, or is the functions are in different spaces """ if MPI.Comm.Compare(f.comm, self.comm) not in {MPI.CONGRUENT, MPI.IDENT}: raise CommMismatchError("Function communicator does not match space communicator") if g is not None: if MPI.Comm.Compare(f.comm, g.comm) not in {MPI.CONGRUENT, MPI.IDENT}: raise CommMismatchError("Mismatching communicators for functions") if f.function_space() != g.function_space(): raise ValueError("Mismatching function spaces for functions")
[docs] @PETSc.Log.EventDecorator() @_ensemble_mpi_dispatch def allreduce(self, f: Function | Cofunction, f_reduced: Function | Cofunction | None = None, op: MPI.Op = MPI.SUM ) -> Function | Cofunction: """ Allreduce a :class:`firedrake.function.Function` ``f`` into ``f_reduced``. Parameters ---------- f : The :class:`firedrake.function.Function` to allreduce. f_reduced : The result of the reduction. Must be in the same :func:`~firedrake.functionspace.FunctionSpace` as ``f``. op : MPI reduction operator. Defaults to MPI.SUM. Returns ------- firedrake.function.Function | Cofunction : The result of the reduction. Raises ------ ValueError If Function communicators mismatch each other or the ensemble spatial communicator, or if the Functions are in different spaces """ f_reduced = f_reduced or Function(f.function_space()) self._check_function(f, f_reduced) with f_reduced.dat.vec_wo as vout, f.dat.vec_ro as vin: self._ensemble_comm.Allreduce(vin.array_r, vout.array, op=op) return f_reduced
[docs] @PETSc.Log.EventDecorator() @_ensemble_mpi_dispatch def iallreduce(self, f: Function | Cofunction, f_reduced: Function | Cofunction | None = None, op: MPI.Op = MPI.SUM ) -> MPI.Request: """ Allreduce (non-blocking) a :class:`firedrake.function.Function` ``f`` into ``f_reduced``. Parameters ---------- f : The a :class:`firedrake.function.Function` to allreduce. f_reduced : The result of the reduction. Must be in the same :func:`~firedrake.functionspace.FunctionSpace` as ``f``. op : MPI reduction operator. Defaults to MPI.SUM. Returns ------- mpi4py.MPI.Request : MPI request. Raises ------ ValueError If Function communicators mismatch each other or the ensemble spatial communicator, or if the Functions are in different spaces """ f_reduced = f_reduced or Function(f.function_space()) self._check_function(f, f_reduced) return self._ensemble_comm.Iallreduce(f.dat.data_ro, f_reduced.dat.data_rw, op=op)
[docs] @PETSc.Log.EventDecorator() @_ensemble_mpi_dispatch def reduce(self, f: Function | Cofunction, f_reduced: Function | Cofunction | None = None, op: MPI.Op = MPI.SUM, root: int = 0 ) -> Function | Cofunction: """ Reduce a :class:`firedrake.function.Function` ``f`` into ``f_reduced``. Parameters ---------- f : The :class:`firedrake.function.Function` to reduce. f_reduced : The result of the reduction. Must be in the same :func:`~firedrake.functionspace.FunctionSpace` as ``f``. op : MPI reduction operator. Defaults to MPI.SUM. root : The ensemble rank to reduce to. Returns ------- firedrake.function.Function | Cofunction : The result of the reduction. Raises ------ ValueError If Function communicators mismatch each other or the ensemble spatial communicator, or if the Functions are in different spaces """ f_reduced = f_reduced or Function(f.function_space()) self._check_function(f, f_reduced) if self.ensemble_comm.rank == root: with f_reduced.dat.vec_wo as vout, f.dat.vec_ro as vin: self._ensemble_comm.Reduce(vin.array_r, vout.array, op=op, root=root) else: with f.dat.vec_ro as vin: self._ensemble_comm.Reduce(vin.array_r, None, op=op, root=root) return f_reduced
[docs] @PETSc.Log.EventDecorator() @_ensemble_mpi_dispatch def ireduce(self, f: Function | Cofunction, f_reduced: Function | Cofunction | None = None, op: MPI.Op = MPI.SUM, root: int = 0 ) -> MPI.Request: """ Reduce (non-blocking) a :class:`firedrake.function.Function` ``f`` into ``f_reduced``. Parameters ---------- f : The a :class:`firedrake.function.Function` to reduce. f_reduced : The result of the reduction. Must be in the same :func:`~firedrake.functionspace.FunctionSpace` as ``f``. op : MPI reduction operator. Defaults to MPI.SUM. root : The ensemble rank to reduce to. Returns ------- mpi4py.MPI.Request MPI request. Raises ------ ValueError If Function communicators mismatch each other or the ensemble spatial communicator, or if the Functions are in different spaces """ f_reduced = f_reduced or Function(f.function_space()) self._check_function(f, f_reduced) return self._ensemble_comm.Ireduce(f.dat.data_ro, f_reduced.dat.data_rw, op=op, root=root)
[docs] @PETSc.Log.EventDecorator() @_ensemble_mpi_dispatch def bcast(self, f: Function | Cofunction, root: int = 0 ) -> Function | Cofunction: """ Broadcast a :class:`firedrake.function.Function` ``f`` over ``ensemble_comm`` from :attr:`~.Ensemble.ensemble_rank` ``root``. Parameters ---------- f : The :class:`firedrake.function.Function` to broadcast. root : The rank to broadcast from. Returns ------- firedrake.function.Function | Cofunction : The result of the broadcast. Raises ------ ValueError If the Function communicator mismatches the ``ensemble.comm``. """ self._check_function(f) with f.dat.vec_rw as vec: self._ensemble_comm.Bcast(vec.array, root=root) return f
[docs] @PETSc.Log.EventDecorator() @_ensemble_mpi_dispatch def ibcast(self, f: Function | Cofunction, root: int = 0 ) -> MPI.Request: """ Broadcast (non-blocking) a :class:`firedrake.function.Function` ``f`` over ``ensemble_comm`` :attr:`~.Ensemble.ensemble_rank` ``root``. Parameters ---------- f : The :class:`firedrake.function.Function` to broadcast. root : The rank to broadcast from. Returns ------- mpi4py.MPI.Request MPI request. Raises ------ ValueError If the Function communicator mismatches the ``ensemble.comm``. """ self._check_function(f) return self._ensemble_comm.Ibcast(f.dat.data_rw, root=root)
[docs] @PETSc.Log.EventDecorator() @_ensemble_mpi_dispatch def send(self, f: Function | Cofunction, dest: int, tag: int = 0): """ Send (blocking) a :class:`firedrake.function.Function` ``f`` over ``ensemble_comm`` to another :attr:`~.Ensemble.ensemble_rank`. Parameters ---------- f : The a :class:`firedrake.function.Function` to send. dest : The :attr:`~.Ensemble.ensemble_rank` to send ``f`` to. tag : The tag of the message. Raises ------ ValueError If the Function communicator mismatches the ``ensemble.comm``. """ self._check_function(f) self._ensemble_comm.Send(f.dat.data_ro, dest=dest, tag=tag)
[docs] @PETSc.Log.EventDecorator() @_ensemble_mpi_dispatch def recv(self, f: Function | Cofunction, source: int = MPI.ANY_SOURCE, tag: int = MPI.ANY_TAG, status: MPI.Status = None, ) -> Function | Cofunction: """ Receive (blocking) a :class:`firedrake.function.Function` ``f`` over ``ensemble_comm`` from another :attr:`~.Ensemble.ensemble_rank`. Parameters ---------- f : The :class:`firedrake.function.Function` to receive into. source : The :attr:`~.Ensemble.ensemble_rank` to receive ``f`` from. tag : The tag of the message. status : The :class:`mpi4py.MPI.Status` of the internal recv call. Returns ------- firedrake.function.Function | Cofunction : ``f`` with the received data. Raises ------ ValueError If the Function communicator mismatches the ``ensemble.comm``. ValueError If the number of ``statuses`` provided is not the number of subfunctions of ``f``. """ self._check_function(f) self._ensemble_comm.Recv(f.dat.data_wo, source=source, tag=tag, status=status) return f
[docs] @PETSc.Log.EventDecorator() @_ensemble_mpi_dispatch def isend(self, f: Function | Cofunction, dest: int, tag: int = 0 ) -> MPI.Request: """ Send (non-blocking) a :class:`firedrake.function.Function` ``f`` over ``ensemble_comm`` to another :attr:`~.Ensemble.ensemble_rank`. Parameters ---------- f : The a :class:`firedrake.function.Function` to send. dest : The :attr:`~.Ensemble.ensemble_rank` to send ``f`` to. tag : The tag of the message. Returns ------- mpi4py.MPI.Request MPI request. Raises ------ ValueError If the Function communicator mismatches the ``ensemble.comm``. """ self._check_function(f) return self._ensemble_comm.Isend(f.dat.data_ro, dest=dest, tag=tag)
[docs] @PETSc.Log.EventDecorator() @_ensemble_mpi_dispatch def irecv(self, f: Function | Cofunction, source: int = MPI.ANY_SOURCE, tag: int = MPI.ANY_TAG ) -> MPI.Request: """ Receive (non-blocking) a :class:`firedrake.function.Function` ``f`` over ``ensemble_comm`` from another :attr:`~.Ensemble.ensemble_rank`. Parameters ---------- f : The :class:`firedrake.function.Function` to receive into. source : The :attr:`~.Ensemble.ensemble_rank` to receive ``f`` from. tag : The tag of the message. Returns ------- mpi4py.MPI.Request MPI request. Raises ------ ValueError If the Function communicator mismatches the ``ensemble.comm``. """ self._check_function(f) return self._ensemble_comm.Irecv(f.dat.data_wo, source=source, tag=tag)
[docs] @PETSc.Log.EventDecorator() @_ensemble_mpi_dispatch def sendrecv(self, fsend: Function | Cofunction, dest: int, sendtag: int = 0, frecv: Function | Cofunction | None = None, source: int = MPI.ANY_SOURCE, recvtag: int = MPI.ANY_TAG, statuses: list[MPI.Status] | MPI.Status = None ) -> Function | Cofunction: """ Send (blocking) a :class:`firedrake.function.Function` ``fsend`` and receive a :class:`firedrake.function.Function` ``frecv`` over ``ensemble_comm`` to/from other :attr:`~.Ensemble.ensemble_rank`. ``fsend`` and ``frecv`` do not need to be in the same function space but do need to have the same number of subfunctions. Parameters ---------- fsend : The a :class:`firedrake.function.Function` to send. dest : The :attr:`~.Ensemble.ensemble_rank` to send ``fsend`` to. sendtag : The tag of the send message. frecv : The :class:`firedrake.function.Function` to receive into. source : The :attr:`~.Ensemble.ensemble_rank` to receive ``frecv`` from. recvtag : The tag of the receive message. statuses : The :class:`mpi4py.MPI.Status` of the internal recv calls (one for each of the ``subfunctions`` of ``frecv``). Returns ------- firedrake.function.Function | Cofunction ``frecv`` with the received data. Raises ------ ValueError If the Function communicators mismatches each other or the ``ensemble.comm``. ValueError If the number of ``statuses`` provided is not the number of subfunctions of ``f``. """ frecv = frecv or Function(fsend.function_space()) # functions don't necessarily have to match self._check_function(fsend) self._check_function(frecv) if statuses is not None and isinstance(statuses, MPI.Status): statuses = [statuses] if statuses is not None and len(statuses) != len(frecv.dat): raise ValueError("Need to provide enough status objects for all parts of the Function") with fsend.dat.vec_ro as sendvec, frecv.dat.vec_wo as recvvec: self._ensemble_comm.Sendrecv(sendvec.array_r, dest, sendtag=sendtag, recvbuf=recvvec.array_w, source=source, recvtag=recvtag, status=statuses) return frecv
[docs] @PETSc.Log.EventDecorator() @_ensemble_mpi_dispatch def isendrecv(self, fsend: Function | Cofunction, dest: int, sendtag: int = 0, frecv: Function | Cofunction | None = None, source: int = MPI.ANY_SOURCE, recvtag: int = MPI.ANY_TAG ) -> list[MPI.Request]: """ Send (non-blocking) a :class:`firedrake.function.Function` ``fsend`` and receive a :class:`firedrake.function.Function` ``frecv`` over ``ensemble_comm`` to/from other :attr:`~.Ensemble.ensemble_rank`. ``fsend`` and ``frecv`` do not need to be in the same function space. Parameters ---------- fsend : The a :class:`firedrake.function.Function` to send. dest : The :attr:`~.Ensemble.ensemble_rank` to send ``fsend`` to. sendtag : The tag of the send message. frecv : The :class:`firedrake.function.Function` to receive into. source : The :attr:`~.Ensemble.ensemble_rank` to receive ``frecv`` from. recvtag : The tag of the receive message. Returns ------- list[mpi4py.MPI.Request] MPI request objects (one for each of fsend and frecv). Raises ------ ValueError If the Function communicators mismatches each other or the ``ensemble.comm``. """ frecv = frecv or Function(fsend.function_space()) # functions don't necessarily have to match self._check_function(fsend) self._check_function(frecv) return [ self._ensemble_comm.Isend(fsend.dat.data_ro, dest=dest, tag=sendtag), self._ensemble_comm.Irecv(frecv.dat.data_wo, source=source, tag=recvtag), ]
[docs] @contextmanager def sequential(self, *, synchronise: bool = False, reverse: bool = False, **kwargs): """ Context manager for executing code on each ensemble member consecutively (ordered by increasing :attr:`~.Ensemble.ensemble_rank`). Any data in ``kwargs`` will be made available in the returned context and will be communicated forward after each ensemble member exits. :class:`firedrake.function.Function` or :class:`.Cofunction` ``kwargs`` will be sent with the corresponding Ensemble methods. For example: .. code-block:: python with ensemble.sequential(index=0) as ctx: print(ensemble.ensemble_rank, ctx.index) ctx.index += 2 Would print: .. code-block:: text 0 0 1 2 2 4 3 6 ... If ``reverse is True`` then the ensemble ranks will be looped through in decreasing order i.e. ``ensemble_rank == (ensemble_size - 1)`` will run first, then ``ensemble_rank == (ensemble_size - 2)`` etc. Parameters ---------- synchronise : If True then MPI_Barrier will be called on the ``global_comm`` at the beginning and end of this method. reverse : If True then will iterate through spatial comms in order of decreasing ``ensemble_rank``. kwargs : Data to be passed forward by each rank and made available in the returned ``ctx``. """ rank = self.ensemble_rank if reverse: # send backwards src = rank + 1 dst = rank - 1 first_rank = (rank == self.ensemble_size - 1) last_rank = (rank == 0) else: # send forwards src = rank - 1 dst = rank + 1 first_rank = (rank == 0) last_rank = (rank == self.ensemble_size - 1) if synchronise: self.global_comm.Barrier() if not first_rank: for i, (k, v) in enumerate(kwargs.items()): if isinstance(v, (Function, Cofunction)): # Functions are sent in-place, everything else is pickled recv_args = [kwargs[k]] else: recv_args = [] kwargs[k] = self.recv(*recv_args, source=src, tag=rank+i*100) ctx = SimpleNamespace(**kwargs) yield ctx if not last_rank: for i, v in enumerate((getattr(ctx, k) for k in kwargs.keys())): try: self.send(v, dest=dst, tag=dst+i*100) except Exception as error: raise TypeError( "Failed to send object of type {type(v)__name__}. kwargs for" " Ensemble.sequential must be Functions, Cofunctions," " or acceptable arguments to mpi4py.MPI.Comm.send." ) from error if synchronise: self.global_comm.Barrier()