parallel.h

Go to the documentation of this file.

// The libMesh Finite Element Library.
// Copyright (C) 2002-2014 Benjamin S. Kirk, John W. Peterson, Roy H. Stogner

// This library is free software; you can redistribute it and/or
// modify it under the terms of the GNU Lesser General Public
// License as published by the Free Software Foundation; either
// version 2.1 of the License, or (at your option) any later version.

// This library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
// Lesser General Public License for more details.

// You should have received a copy of the GNU Lesser General Public
// License along with this library; if not, write to the Free Software
// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA


#ifndef LIBMESH_PARALLEL_H
#define LIBMESH_PARALLEL_H

// Local includes
#include "libmesh/libmesh_common.h" // libmesh_assert, cast_int
#include "libmesh/libmesh_logging.h"
#include "libmesh/auto_ptr.h"

// C++ includes
#include <cstddef>
#include <climits>
#include <iterator>
#include <limits>
#include <map>
#include <set>
#include <string>
#include <vector>

namespace libMesh
{


// Macro to identify and debug functions which should only be called in
// parallel on every processor at once

#undef parallel_only
#ifndef NDEBUG
#define parallel_only() do {                                            \
    libmesh_deprecated();                                               \
    libmesh_assert(CommWorld.verify(std::string(__FILE__).size()));     \
    libmesh_assert(CommWorld.verify(std::string(__FILE__)));            \
    libmesh_assert(CommWorld.verify(__LINE__)); } while (0)
#else
#define parallel_only()  ((void) 0)
#endif

#undef libmesh_parallel_only
#ifndef NDEBUG
#define libmesh_parallel_only(comm_obj) do {                            \
    libmesh_assert((comm_obj).verify(std::string(__FILE__).size()));    \
    libmesh_assert((comm_obj).verify(std::string(__FILE__)));           \
    libmesh_assert((comm_obj).verify(__LINE__)); } while (0)
#else
#define libmesh_parallel_only(comm_obj)  ((void) 0)
#endif

// Macro to identify and debug functions which should only be called in
// parallel on every processor at once

#undef parallel_only_on
#ifndef NDEBUG
#define parallel_only_on(comm_arg) do {                                 \
    libmesh_deprecated();                                               \
    libmesh_assert(CommWorld.verify(std::string(__FILE__).size(), comm_arg)); \
    libmesh_assert(CommWorld.verify(std::string(__FILE__), comm_arg));  \
    libmesh_assert(CommWorld.verify(__LINE__), comm_arg); } while (0)
#else
#define parallel_only_on(comm_arg)  ((void) 0)
#endif

#undef libmesh_parallel_only_on
#ifndef NDEBUG
#define libmesh_parallel_only_on(comm_obj,comm_arg) do {                \
    libmesh_assert(comm_obj.verify(std::string(__FILE__).size(), comm_arg)); \
    libmesh_assert(comm_obj.verify(std::string(__FILE__), comm_arg));   \
    libmesh_assert(comm_obj.verify(__LINE__), comm_arg); } while (0)
#else
#define libmesh_parallel_only_on(comm_obj,comm_arg)  ((void) 0)
#endif

namespace Parallel
{
//-------------------------------------------------------------------
class Communicator;
class DataType;
class Request;
class Status;

#ifdef LIBMESH_HAVE_MPI
//-------------------------------------------------------------------
typedef MPI_Datatype data_type;

typedef MPI_Request request;

typedef MPI_Status status;

typedef MPI_Comm communicator;

template <typename T>
inline data_type dataplusint_type();

template <typename T>
class DataPlusInt
{
public:
  T val;
  int rank;
};

const unsigned int any_source =
  static_cast<unsigned int>(MPI_ANY_SOURCE);


#else

// These shouldn't actually be needed, but must be
// unique types for function overloading to work
// properly.
struct data_type    { /* unsigned int t; */ };
struct request      { /* unsigned int r; */ };
struct status       { /* unsigned int s; */ };
typedef int communicator; // Must match petsc-nompi definition

const unsigned int any_source=0;
#endif // LIBMESH_HAVE_MPI



//-------------------------------------------------------------------
class MessageTag
{
public:

  static const int invalid_tag = INT_MIN;

  explicit MessageTag(int tagvalue = invalid_tag)
    : _tagvalue(tagvalue), _comm(NULL) {}

  MessageTag(const MessageTag& other);

  ~MessageTag();

  int value() const {
    return _tagvalue;
  }

private:
  int _tagvalue;
  const Communicator *_comm;

  // Constructor for reference-counted unique tags
  MessageTag(int tagvalue, const Communicator *comm)
    : _tagvalue(tagvalue), _comm(comm) {}

  // Let Communicator handle the reference counting
  friend class Communicator;
};


//-------------------------------------------------------------------
#ifdef LIBMESH_HAVE_MPI
const MessageTag any_tag = MessageTag(MPI_ANY_TAG);
#else
const MessageTag any_tag = MessageTag(-1);
#endif

const MessageTag no_tag = MessageTag(0);


//-------------------------------------------------------------------
class DataType
{
public:
  DataType () : _datatype() {}

  DataType (const DataType &other) :
    _datatype(other._datatype)
  {}

  DataType (const data_type &type) :
    _datatype(type)
  {}

#ifdef LIBMESH_HAVE_MPI
  DataType (const DataType &other, unsigned int count)
  {
    MPI_Type_contiguous(count, other._datatype, &_datatype);
    this->commit();
  }
#else
  DataType (const DataType &, unsigned int)
  {
  }
#endif

  DataType & operator = (const DataType &other)
  { _datatype = other._datatype; return *this; }

  DataType & operator = (const data_type &type)
  { _datatype = type; return *this; }

  operator const data_type & () const
  { return _datatype; }

  operator data_type & ()
  { return _datatype; }

  //     operator data_type const * () const
  //     { return &_datatype; }

  //     operator data_type * ()
  //     { return &_datatype; }

  void commit ()
  {
#ifdef LIBMESH_HAVE_MPI
    MPI_Type_commit (&_datatype);
#endif
  }

  void free ()
  {
#ifdef LIBMESH_HAVE_MPI
    MPI_Type_free (&_datatype);
#endif
  }

protected:

  data_type _datatype;
};


//-------------------------------------------------------------------

#ifdef LIBMESH_HAVE_CXX11
// A C++03-compatible replacement for std::false_type
struct false_type
{
  static const bool value = false;
  typedef bool value_type;
  typedef false_type type;
  operator value_type() const { return value; }
};

// Templated helper class to be used with static_assert.
template<typename T>
struct dependent_false : false_type
{};
#endif

template <typename T>
class StandardType : public DataType
{
#ifdef LIBMESH_HAVE_CXX11
  // Get a slightly better compiler diagnostic if we have C++11
  static_assert(dependent_false<T>::value,
                "Only specializations of StandardType may be used, did you forget to include a header file (e.g. parallel_algebra.h)?");
#endif

  /*
   * The unspecialized class is useless, so we make its constructor
   * private to catch mistakes at compile-time rather than link-time.
   * Specializations should have a public constructor of the same
   * form.
   */
private:
  StandardType(const T* example = NULL);
};

/*
 * The unspecialized class gives default, lowest-common-denominator
 * attributes, for values which can't be used with Parallel min/max.
 * Specialized classes can set this to true, and should define
 * the lowest and highest values possible for the type.
 */
template<typename T>
struct Attributes
{
  static const bool has_min_max = false;
  static void set_lowest(T&) {}
  static void set_highest(T&) {}
};



//-------------------------------------------------------------------
class Status
{
public:
  Status ();

  explicit Status (const data_type &type);

  explicit Status (const status &status);

  Status (const status    &status,
          const data_type &type);

  Status (const Status &status);

  Status (const Status    &status,
          const data_type &type);

  status * get() { return &_status; }

  status const * get() const { return &_status; }

  int source () const;

  int tag () const;

  data_type& datatype () { return _datatype; }

  const data_type& datatype () const { return _datatype; }

  unsigned int size (const data_type &type) const;

  unsigned int size () const;

private:

  status    _status;
  data_type _datatype;
};


//-------------------------------------------------------------------
struct PostWaitWork {
  virtual ~PostWaitWork() {}

  virtual void run() {}
};


//-------------------------------------------------------------------
class Request
{
public:
  Request ();

  Request (const request &r);

  Request (const Request &other);

  void cleanup();

  Request & operator = (const Request &other);

  Request & operator = (const request &r);

  ~Request ();

  request* get() { return &_request; }

  const request* get() const { return &_request; }

  Status wait ();

  bool test ();

  bool test (status &status);

  void add_prior_request(const Request& req);

  void add_post_wait_work(PostWaitWork* work);

private:
  request _request;

  // Breaking non-blocking sends into multiple requests can require chaining
  // multiple requests into a single Request
  UniquePtr<Request> _prior_request;

  // post_wait_work->first is a vector of work to do after a wait
  // finishes; post_wait_work->second is a reference count so that
  // Request objects will behave roughly like a shared_ptr and be
  // usable in STL containers
  std::pair<std::vector <PostWaitWork* >, unsigned int>* post_wait_work;
};

inline Status wait (Request &r) { return r.wait(); }

inline void wait (std::vector<Request> &r)
{ for (unsigned int i=0; i<r.size(); i++) r[i].wait(); }


template <typename T>
struct BufferType;

template <typename T>
struct BufferType<T*> {
  typedef typename BufferType<const T*>::type type;
};

template <typename T, typename buffertype, typename Context>
void pack(const T* object,
          typename std::vector<buffertype>& data,
          const Context* context);

template <typename T, typename Context>
unsigned int packable_size(const T*, const Context*);

template <typename T, typename BufferIter>
unsigned int packed_size(const T*,
                         BufferIter);

template <typename T, typename BufferIter, typename Context>
void unpack(BufferIter in, T** out, Context* ctx);

template <typename Context, typename buffertype, typename OutputIter>
inline void unpack_range (const typename std::vector<buffertype>& buffer,
                          Context *context,
                          OutputIter out);

template <typename Context, typename buffertype, typename Iter>
inline Iter pack_range (const Context *context,
                        Iter range_begin,
                        const Iter range_end,
                        typename std::vector<buffertype>& buffer);

template <typename Context, typename Iter>
inline std::size_t packed_range_size (const Context *context,
                                      Iter range_begin,
                                      const Iter range_end);

//-------------------------------------------------------------------
class Communicator
{
  // Basic operations:
public:

  Communicator ();

  /*
   * Constructor from MPI_Comm
   */
  explicit Communicator (const communicator &comm);

  /*
   * NON-VIRTUAL destructor
   */
  ~Communicator ();

  /*
   * Create a new communicator between some subset of \p this
   */
  void split(int color, int key, Communicator &target) const;

  /*
   * Create a new duplicate of \p this communicator
   */
  void duplicate(const Communicator &comm);

  /*
   * Create a new duplicate of an MPI communicator
   */
  void duplicate(const communicator &comm);

  communicator& get() { return _communicator; }

  const communicator& get() const { return _communicator; }

  MessageTag get_unique_tag(int tagvalue) const;

  void reference_unique_tag(int tagvalue) const;

  void dereference_unique_tag(int tagvalue) const;

  void clear();

  Communicator& operator= (const communicator &comm);

  unsigned int rank() const { return _rank; }

  unsigned int size() const { return _size; }

  enum SendMode { DEFAULT=0, SYNCHRONOUS };

private:

  // Don't use the copy constructor, just copy by reference or
  // pointer - it's too hard to keep a common used_tag_values if
  // each communicator is shared by more than one Communicator
  explicit Communicator (const Communicator &);

  void assign(const communicator &comm);

  communicator  _communicator;
  unsigned int  _rank, _size;
  SendMode _send_mode;

  // mutable used_tag_values - not thread-safe, but then Parallel::
  // isn't thread-safe in general.
  mutable std::map<int, unsigned int> used_tag_values;
  bool          _I_duped_it;

  // Communication operations:
public:

  void send_mode (const SendMode sm) { _send_mode = sm; }

  SendMode send_mode() const { return _send_mode; }

  void barrier () const;

  template <typename T>
  bool verify(const T &r) const;

  template <typename T>
  bool semiverify(const T *r) const;

  template <typename T>
  void min(T &r) const;

  template <typename T>
  void minloc(T &r,
              unsigned int &min_id) const;

  template <typename T>
  void minloc(std::vector<T> &r,
              std::vector<unsigned int> &min_id) const;

  template <typename T>
  void max(T &r) const;

  template <typename T>
  void maxloc(T &r,
              unsigned int &max_id) const;

  template <typename T>
  void maxloc(std::vector<T> &r,
              std::vector<unsigned int> &max_id) const;

  template <typename T>
  void sum(T &r) const;

  template <typename T>
  void set_union(T &data, const unsigned int root_id) const;

  template <typename T>
  void set_union(T &data) const;

  status probe (const unsigned int src_processor_id,
                const MessageTag &tag=any_tag) const;

  template <typename T>
  void send (const unsigned int dest_processor_id,
             T &buf,
             const MessageTag &tag=no_tag) const;

  template <typename T>
  void send (const unsigned int dest_processor_id,
             T &buf,
             Request &req,
             const MessageTag &tag=no_tag) const;

  template <typename T>
  void send (const unsigned int dest_processor_id,
             T &buf,
             const DataType &type,
             const MessageTag &tag=no_tag) const;

  template <typename T>
  void send (const unsigned int dest_processor_id,
             T &buf,
             const DataType &type,
             Request &req,
             const MessageTag &tag=no_tag) const;

  template <typename T>
  Status receive (const unsigned int dest_processor_id,
                  T &buf,
                  const MessageTag &tag=any_tag) const;

  template <typename T>
  void receive (const unsigned int dest_processor_id,
                T &buf,
                Request &req,
                const MessageTag &tag=any_tag) const;

  template <typename T>
  Status receive (const unsigned int dest_processor_id,
                  T &buf,
                  const DataType &type,
                  const MessageTag &tag=any_tag) const;

  template <typename T>
  void receive (const unsigned int dest_processor_id,
                T &buf,
                const DataType &type,
                Request &req,
                const MessageTag &tag=any_tag) const;

  template <typename Context, typename Iter>
  void send_packed_range (const unsigned int dest_processor_id,
                          const Context *context,
                          Iter range_begin,
                          const Iter range_end,
                          const MessageTag &tag=no_tag) const;

  template <typename Context, typename Iter>
  void send_packed_range (const unsigned int dest_processor_id,
                          const Context *context,
                          Iter range_begin,
                          const Iter range_end,
                          Request &req,
                          const MessageTag &tag=no_tag) const;

  template <typename Context, typename OutputIter>
  void receive_packed_range (const unsigned int dest_processor_id,
                             Context *context,
                             OutputIter out,
                             const MessageTag &tag=any_tag) const;

  // template <typename Context, typename OutputIter>
  // void receive_packed_range (const unsigned int dest_processor_id,
  //                            Context *context,
  //                            OutputIter out,
  //                            Request &req,
  //                            const MessageTag &tag=any_tag) const;

  template <typename T1, typename T2>
  void send_receive(const unsigned int dest_processor_id,
                    T1 &send,
                    const unsigned int source_processor_id,
                    T2 &recv,
                    const MessageTag &send_tag = no_tag,
                    const MessageTag &recv_tag = any_tag) const;

  template <typename Context1, typename RangeIter, typename Context2, typename OutputIter>
  void send_receive_packed_range(const unsigned int dest_processor_id,
                                 const Context1* context1,
                                 RangeIter send_begin,
                                 const RangeIter send_end,
                                 const unsigned int source_processor_id,
                                 Context2* context2,
                                 OutputIter out,
                                 const MessageTag &send_tag = no_tag,
                                 const MessageTag &recv_tag = any_tag) const;

  template <typename T1, typename T2>
  void send_receive(const unsigned int dest_processor_id,
                    T1 &send,
                    const DataType &type1,
                    const unsigned int source_processor_id,
                    T2 &recv,
                    const DataType &type2,
                    const MessageTag &send_tag = no_tag,
                    const MessageTag &recv_tag = any_tag) const;

  template <typename T>
  inline void gather(const unsigned int root_id,
                     T send,
                     std::vector<T> &recv) const;

  template <typename T>
  inline void gather(const unsigned int root_id,
                     std::vector<T> &r) const;

  template <typename T>
  inline void allgather(T send,
                        std::vector<T> &recv) const;


  template <typename T>
  inline void allgather(std::vector<T> &r,
                        const bool identical_buffer_sizes = false) const;

  //-------------------------------------------------------------------
  template <typename Context, typename Iter, typename OutputIter>
  inline void gather_packed_range (const unsigned int root_id,
                                   Context *context,
                                   Iter range_begin,
                                   const Iter range_end,
                                   OutputIter out) const;

  template <typename Context, typename Iter, typename OutputIter>
  inline void allgather_packed_range (Context *context,
                                      Iter range_begin,
                                      const Iter range_end,
                                      OutputIter out) const;

  template <typename T>
  inline void alltoall(std::vector<T> &r) const;

  template <typename T>
  inline void broadcast(T &data, const unsigned int root_id=0) const;

  template <typename Context, typename OutputContext, typename Iter, typename OutputIter>
  inline void broadcast_packed_range (const Context *context1,
                                      Iter range_begin,
                                      const Iter range_end,
                                      OutputContext *context2,
                                      OutputIter out,
                                      const unsigned int root_id = 0) const;

#include "libmesh/parallel_communicator_specializations"

}; // class Communicator

// FakeCommunicator for debugging inappropriate CommWorld uses
class FakeCommunicator
{
  operator Communicator& ()
  {
    libmesh_not_implemented();
    static Communicator temp;
    return temp;
  }
};

// PostWaitWork specialization for copying from temporary to
// output containers
template <typename Container, typename OutputIter>
struct PostWaitCopyBuffer : public PostWaitWork {
  PostWaitCopyBuffer(const Container& buffer, const OutputIter out)
    : _buf(buffer), _out(out) {}

  virtual void run() { std::copy(_buf.begin(), _buf.end(), _out); }

private:
  const Container& _buf;
  OutputIter _out;
};

// PostWaitWork specialization for unpacking received buffers.
template <typename Container, typename Context, typename OutputIter>
struct PostWaitUnpackBuffer : public PostWaitWork {
  PostWaitUnpackBuffer(const Container& buffer, Context *context, OutputIter out) :
    _buf(buffer), _context(context), _out(out) {}

  virtual void run() { Parallel::unpack_range(_buf, _context, _out); }

private:
  const Container& _buf;
  Context *_context;
  OutputIter _out;
};


// PostWaitWork specialization for freeing no-longer-needed buffers.
template <typename Container>
struct PostWaitDeleteBuffer : public PostWaitWork {
  PostWaitDeleteBuffer(Container* buffer) : _buf(buffer) {}

  virtual void run() { delete _buf; }

private:
  Container* _buf;
};

} // namespace Parallel

#ifdef LIBMESH_DISABLE_COMMWORLD
extern Parallel::FakeCommunicator CommWorld;
#define LIBMESH_CAN_DEFAULT_TO_COMMWORLD
#else
extern Parallel::Communicator CommWorld;
#define LIBMESH_CAN_DEFAULT_TO_COMMWORLD = libMesh::CommWorld
#endif

} // namespace libMesh

// Define all the implementations separately; users might want to look
// through this file for APIs, and it's long enough already.

#include "libmesh/parallel_implementation.h"

#endif // LIBMESH_PARALLEL_H