communicator.h

Go to the documentation of this file.

//LIC// ====================================================================
//LIC// This file forms part of oomph-lib, the object-oriented, 
//LIC// multi-physics finite-element library, available 
//LIC// at http://www.oomph-lib.org.
//LIC// 
//LIC//    Version 1.0; svn revision $LastChangedRevision$
//LIC//
//LIC// $LastChangedDate$
//LIC// 
//LIC// Copyright (C) 2006-2016 Matthias Heil and Andrew Hazel
//LIC// 
//LIC// This library is free software; you can redistribute it and/or
//LIC// modify it under the terms of the GNU Lesser General Public
//LIC// License as published by the Free Software Foundation; either
//LIC// version 2.1 of the License, or (at your option) any later version.
//LIC// 
//LIC// This library is distributed in the hope that it will be useful,
//LIC// but WITHOUT ANY WARRANTY; without even the implied warranty of
//LIC// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
//LIC// Lesser General Public License for more details.
//LIC// 
//LIC// You should have received a copy of the GNU Lesser General Public
//LIC// License along with this library; if not, write to the Free Software
//LIC// Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
//LIC// 02110-1301  USA.
//LIC// 
//LIC// The authors may be contacted at oomph-lib@maths.man.ac.uk.
//LIC// 
//LIC//====================================================================
#ifndef OOMPH_COMMUNICATOR_CLASS_HEADER
#define OOMPH_COMMUNICATOR_CLASS_HEADER

// Config header generated by autoconfig
#ifdef HAVE_CONFIG_H
  #include <oomph-lib-config.h>
#endif

// MPI headers
#ifdef OOMPH_HAS_MPI
#include "mpi.h"
#endif

// Oomph-lib error handler
#include "oomph_definitions.h"
#include "oomph_utilities.h"

namespace oomph{

template <class T>
class DenseMatrix;

//=============================================================================
/// \short An oomph-lib wrapper to the MPI_Comm communicator object. Just 
/// contains an MPI_Comm object (which is a pointer) and wrappers to
/// the MPI_... methods.
//=============================================================================
class OomphCommunicator
{

  public:

#ifdef OOMPH_HAS_MPI
 /// Construct a communicator from a MPI_Comm object. 
 /// if the bool owns_mpi_comm is true then this communicator is responsible
 /// for the destruction of the mpi_communicator. The mpi comm will be freed
 /// when the destructor is called.
 OomphCommunicator(const MPI_Comm mpi_communicator, 
                   const bool& owns_mpi_comm = false)
  : Serial_communicator(false)
  {
   // store a pointer to the communicator
   Comm = mpi_communicator;
   
   // hold owns_mpi_comm
   Owns_mpi_comm = owns_mpi_comm;
  }
#endif

 /// Serial constructor
 OomphCommunicator()
#ifdef OOMPH_HAS_MPI
  :  Owns_mpi_comm(false), Serial_communicator(true)
#endif
  {}

 /// Copy constructor.
 OomphCommunicator(const OomphCommunicator& communicator)
#ifdef OOMPH_HAS_MPI
  : Owns_mpi_comm(false)
  {
   if (communicator.serial_communicator())
    {
     Serial_communicator = true;
    }
   else
    {
     Comm = communicator.mpi_comm();
     Serial_communicator = false;
    }
  }
#else
 {}
#endif

 /// Pointer (copy) constructor.
 OomphCommunicator(const OomphCommunicator* communicator_pt)
#ifdef OOMPH_HAS_MPI
  : Owns_mpi_comm(false)
  {
   if (communicator_pt->serial_communicator())
    {
     Serial_communicator = true;
    }
   else
    {
     Comm = communicator_pt->mpi_comm();
     Serial_communicator = false;
    }
  }
#else
 {}
#endif

 /// \short Destructor. If MPI and this preconditioner owns the MPI_Comm 
 /// object then MPI_Comm_free is called, otherwise nothing happens as
 /// the destruction of the underlying MPI_Comm object is the responsibility
 /// of another communicator.
 ~OomphCommunicator()
  {
#ifdef OOMPH_HAS_MPI
   if (Owns_mpi_comm)
    {
     MPI_Comm_free(&Comm);
    }
#endif
  }

 /// assignment operator
 void operator=(const OomphCommunicator& communicator)
   {
#ifdef OOMPH_HAS_MPI
     if (Owns_mpi_comm)
       {
         MPI_Comm_free(&Comm);
       }
     Owns_mpi_comm = false;
     if (communicator.serial_communicator())
      {
       Serial_communicator = true;
      }
     else
      {
       Serial_communicator = false;
       Comm = communicator.mpi_comm();
      }
#endif
   }
   
 /// number of processors
 int nproc() const
  {
#ifdef OOMPH_HAS_MPI
   if (Serial_communicator)
    {
     return 1;
    }
   else
    {
     int n_proc=1;
     MPI_Comm_size(Comm,&n_proc);
     return n_proc;
    }
#else
   return 1;
#endif
  }

 /// my rank
 int my_rank() const
  {
#ifdef OOMPH_HAS_MPI
   if (Serial_communicator)
    {
     return 0;
    }
   else
    {
     int My_rank=0;
     MPI_Comm_rank(Comm,&My_rank);
     return My_rank;
    }
#else
   return 0;
#endif
  }

 /// \short == operator - only returns true if communicators are MPI_IDENT,
 /// i.e. if both group and context are the same
 bool operator==(const OomphCommunicator& other_comm) const
  {
#ifdef OOMPH_HAS_MPI
   if (Serial_communicator != other_comm.serial_communicator())
    {
     return false;
    }
   else if (Serial_communicator)
    {
     return true;
    }
   else
    {
     int flag;
     MPI_Comm_compare(Comm,other_comm.mpi_comm(),&flag);
     if (flag == MPI_IDENT)
      {
       return true;
      }
     return false;
    }
#else
   return true;
#endif
  }

 /// \short != operator returns !(==operator) (see ==operator for more details)
 bool operator!=(const OomphCommunicator& other_comm) const
  {
   return !(*this == other_comm);
  }

#ifdef OOMPH_HAS_MPI
 /// \short split the communicator: color is an integer label for the sub
 /// group of processors. key is the rank in this sub group
 OomphCommunicator* split(const int& color, const int& key)
  {
#ifdef PARANOID
   if (Serial_communicator)
    {
     std::ostringstream error_message_stream;                           
     error_message_stream                                        
      << "Attempted to split a serial communicator.";               
     throw OomphLibError(error_message_stream.str(),
                         OOMPH_CURRENT_FUNCTION,
                         OOMPH_EXCEPTION_LOCATION);         
    }
#endif

   // pointer for the split oomph-lib communicator
   OomphCommunicator* split_comm_pt;
   
   // the pointer for the new mpi communicator
   MPI_Comm* mpi_comm_pt = new MPI_Comm;

   // get the split communicator
   MPI_Comm_split(Comm,color,key,mpi_comm_pt);

   // assemble the new oomph-lib ocmmunicator
   // the new oomph-lib communicator owns the MPI_Comm at mpi_comm_pt
   // and is responsible for its destruction
   split_comm_pt = new OomphCommunicator(*mpi_comm_pt,true);

   // and return
   return split_comm_pt;
  }
#endif


#ifdef OOMPH_HAS_MPI
 /// access function to the underlying MPI_Comm object
 MPI_Comm mpi_comm() const 
  {
#ifdef PARANOID
   if (Serial_communicator)
    {
     std::ostringstream error_message_stream;                           
     error_message_stream                                        
      << "Requested the MPI_Comm object for a serial communicator.";
     throw OomphLibError(error_message_stream.str(),
                         OOMPH_CURRENT_FUNCTION,
                         OOMPH_EXCEPTION_LOCATION);         
    }
#endif
   return Comm;
  }
 

 /// Access function (read only) to the serial communicator flag
 bool serial_communicator() const
  {
   return Serial_communicator;
  }

 /// broadcast function for Vector<int>
 void broadcast(const int& source, Vector<int>& x)
  {
   int n;

   // Get number of entries on processor source (where the vector exists)
   unsigned long n_long;
   if (this->my_rank()==source)
    {
     n_long=x.size();

     // Convert to int
     n=static_cast<int>(n_long);
    }
 
   // Broadcast to everybody how many entries to expect
   MPI_Bcast(&n,1,MPI_INT,source,this->mpi_comm());

   // Resize Vector everywhere else in preparation
   if (this->my_rank()!=source)
    {
     x.resize(n);
    }

   // Broadcast the Vector directly
   MPI_Bcast(&x[0],n,MPI_INT,source,this->mpi_comm());
  }

 /// broadcast function for Vector<double>
 void broadcast(const int& source, Vector<double>& x)
  {
   int n;

   // Get number of entries on processor source (where the vector exists)
   unsigned long n_long;
   if (this->my_rank()==source)
    {
     n_long=x.size();

     // Convert to int
     n=static_cast<int>(n_long);
    }
 
   // Broadcast to everybody how many entries to expect
   MPI_Bcast(&n,1,MPI_INT,source,this->mpi_comm());

   // Resize Vector everywhere else in preparation
   if (this->my_rank()!=source)
    {
     x.resize(n);
    }

   // Broadcast the Vector directly
   MPI_Bcast(&x[0],n,MPI_DOUBLE,source,this->mpi_comm());
  }

 /// broadcast function for DenseMatrix<double>
 void broadcast(const int& source, DenseMatrix<double>& x);

#endif

  private:

#ifdef OOMPH_HAS_MPI
 /// the MPI_Comm communicator
 MPI_Comm Comm;

 /// \short boolean indiacting whether this communicator owns the underlying 
 /// MPI_Comm - if so the destructor will free
 /// the communicator
 bool Owns_mpi_comm;

 /// \short boolean to indicate if this communicator is for serial problems.
 /// This is true when serial codes are compiled under MPI
 bool Serial_communicator;

#endif
}; // end of class Communicator
} // end of oomph namespace
#endif