missing_masters.cc

Go to the documentation of this file.

//LIC// ====================================================================
//LIC// This file forms part of oomph-lib, the object-oriented, 
//LIC// multi-physics finite-element library, available 
//LIC// at http://www.oomph-lib.org.
//LIC// 
//LIC//    Version 1.0; svn revision $LastChangedRevision$
//LIC//
//LIC// $LastChangedDate$
//LIC// 
//LIC// Copyright (C) 2006-2016 Matthias Heil and Andrew Hazel
//LIC// 
//LIC// This library is free software; you can redistribute it and/or
//LIC// modify it under the terms of the GNU Lesser General Public
//LIC// License as published by the Free Software Foundation; either
//LIC// version 2.1 of the License, or (at your option) any later version.
//LIC// 
//LIC// This library is distributed in the hope that it will be useful,
//LIC// but WITHOUT ANY WARRANTY; without even the implied warranty of
//LIC// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
//LIC// Lesser General Public License for more details.
//LIC// 
//LIC// You should have received a copy of the GNU Lesser General Public
//LIC// License along with this library; if not, write to the Free Software
//LIC// Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
//LIC// 02110-1301  USA.
//LIC// 
//LIC// The authors may be contacted at oomph-lib@maths.man.ac.uk.
//LIC// 
//LIC//====================================================================
//Non-templated missing masters functions which help to reconcile the hanging
//status of nodes in the halo layer of distributed meshes

//oomph-lib header
#include "missing_masters.h"
#include "missing_masters.template.cc"
#include "mesh.h"
#include "algebraic_elements.h"
#include "macro_element_node_update_element.h"
#include "Qelements.h"

namespace oomph
{

//======================================================================
// Namespace for "global" missing-master-locating functions
//======================================================================
namespace Missing_masters_functions
 {


  // Workspace for locate zeta methods
  //----------------------------------

#ifdef ANNOTATE_MISSING_MASTERS_COMMUNICATION

  // Temporary vector of strings to enable full annotation of multi domain
  // comms (but keep alive because it would be such a bloody pain to
  // rewrite it if things ever go wrong again...)
  // This is left over from the multi-domain stuff and should work
  // in the same way, but it has not been tested.
  Vector<std::string> Flat_packed_unsigneds_string;

#endif

  /// \short Boolean to indicate whether to doc timings or not.
  bool Doc_timings=false;

  /// \short Boolean to indicate whether to output basic info during
  ///        setup_multi_domain_interaction() routines
  bool Doc_stats=false;

  /// \short Boolean to indicate whether to output further info during
  ///        setup_multi_domain_interaction() routines
  bool Doc_full_stats=false;

#ifdef OOMPH_HAS_MPI


  // Functions for location method in multi-domain problems 

//========start of add_external_haloed_node_to_storage====================
/// Helper function to add external haloed nodes, including any masters
//========================================================================
 void add_external_haloed_node_to_storage(int& iproc, Node* nod_pt,
                                          Mesh* const &mesh_pt,
                                          int& n_cont_inter_values,
                                          Vector<unsigned>& send_unsigneds,
                                          Vector<double>& send_doubles)
 {
  // Add the node if required
  add_external_haloed_node_helper(iproc,nod_pt,mesh_pt,
                                  n_cont_inter_values,
                                  send_unsigneds,
                                  send_doubles);
  
  // Recursively add any master nodes (and their master nodes etc)
  recursively_add_masters_of_external_haloed_node(iproc, nod_pt,
                                                  mesh_pt,
                                                  n_cont_inter_values,
                                                  send_unsigneds,
                                                  send_doubles);
 }
  
  
  
 //========================================================================
 ///Recursively add any master nodes (and their master nodes etc) of
 /// external nodes
 //========================================================================
  void recursively_add_masters_of_external_haloed_node(
   int& iproc, Node* nod_pt,
   Mesh* const &mesh_pt,
   int& n_cont_inter_values,
   Vector<unsigned>& send_unsigneds,
   Vector<double>& send_doubles)
  {
   
   // Loop over continuously interpolated values and add masters
   for (int i_cont=-1;i_cont<n_cont_inter_values;i_cont++)
    {
     if (nod_pt->is_hanging(i_cont))
      {
       // Indicate that this node is a hanging node so the other
       // process knows to create HangInfo and masters, etc.
       send_unsigneds.push_back(1);
#ifdef ANNOTATE_MISSING_MASTERS_COMMUNICATION
       Flat_packed_unsigneds_string.push_back("Is hanging");
#endif
       // If this is a hanging node then add all its masters as
       // external halo nodes if they have not yet been added
       HangInfo* hang_pt=nod_pt->hanging_pt(i_cont);
       // Loop over masters
       unsigned n_master=hang_pt->nmaster();
       
       // Indicate number of master nodes to add on other process
       send_unsigneds.push_back(n_master);
#ifdef ANNOTATE_MISSING_MASTERS_COMMUNICATION
       Flat_packed_unsigneds_string.push_back("nmaster");
#endif
       for (unsigned m=0;m<n_master;m++)
        {
         Node* master_nod_pt=hang_pt->master_node_pt(m);
         
         // Call the helper function for master nodes
         add_external_haloed_master_node_helper(iproc,master_nod_pt,
                                                mesh_pt,
                                                n_cont_inter_values,
                                                send_unsigneds,
                                                send_doubles);
         
         // Indicate the weight of this master
         send_doubles.push_back(hang_pt->master_weight(m));

         // Recursively add any master nodes (and their master nodes etc)
         recursively_add_masters_of_external_haloed_node(iproc, master_nod_pt,
                                                         mesh_pt,
                                                         n_cont_inter_values,
                                                         send_unsigneds,
                                                         send_doubles);
        }
      }
     else
      {       
       // Indicate that it's not a hanging node in this variable
       send_unsigneds.push_back(0);
#ifdef ANNOTATE_MISSING_MASTERS_COMMUNICATION
       Flat_packed_unsigneds_string.push_back("Not hanging");
#endif
      }
    } // end loop over continously interpolated values
   
  }
  
//==========start of add_external_haloed_node_helper======================
/// Helper to add external haloed node that is not a master
//========================================================================
  void add_external_haloed_node_helper(int& iproc, Node* nod_pt,
                                       Mesh* const &mesh_pt,
                                       int& n_cont_inter_values,
                                       Vector<unsigned>& send_unsigneds,
                                       Vector<double>& send_doubles)
  {
   //Check to see if this haloed node already exists in (internal)
   //haloed node storage with any processor
   bool found_internally = false;
   unsigned shared_node_index=0;

   //Get vector of all shared nodes with processor iproc
   Vector<Node*> shared_node_pt;
   mesh_pt->get_shared_node_pt(iproc,shared_node_pt);
 
   //Search the internal haloed storage for this node
   Vector<Node*>::iterator it
    = std::find(shared_node_pt.begin(),
                shared_node_pt.end(),
                nod_pt);
   
   //Check if the node was found in shared storage
   if(it != shared_node_pt.end())
    {
     //Node found in (internal) haloed storage
     found_internally = true;
     //Store the index in this storage
     shared_node_index = it - shared_node_pt.begin();
    }

   //// Slow search version without additional access function in Mesh class
   ////Search the internal shared node storage for this node
   //for(unsigned i=0; i<mesh_pt->nshared_node(iproc); i++)
   // {
   //  if(nod_pt == mesh_pt->shared_node_pt(iproc,i))
   //   {
   //    //Node found in (internal) shared storage
   //    found_internally = true;
   //    shared_node_index = i;
   //    break;
   //   }
   // }

   // If we've found the node internally
   if(found_internally)
    {
     // Indicate that this node doesn not need to be constructed on
     // the other process
     send_unsigneds.push_back(0);
#ifdef ANNOTATE_MISSING_MASTERS_COMMUNICATION
     std::stringstream junk;
     junk << "Node was already added [size=" 
          << send_unsigneds.size() << "]; last entry: "
          << send_unsigneds[send_unsigneds.size()-1];

     Flat_packed_unsigneds_string.push_back(junk.str());
#endif

     // This node is already shared with processor iproc, so tell the other
     // processor its index in the shared node storage
     send_unsigneds.push_back(1);
#ifdef ANNOTATE_MISSING_MASTERS_COMMUNICATION
     Flat_packed_unsigneds_string.push_back("haloed node found internally");
#endif
     send_unsigneds.push_back(shared_node_index);
#ifdef ANNOTATE_MISSING_MASTERS_COMMUNICATION
     Flat_packed_unsigneds_string.push_back("(internal) haloed node index");
#endif
    }
   else
    {
     // Attempt to add this node as an external haloed node
     unsigned n_ext_haloed_nod=mesh_pt->nexternal_haloed_node(iproc);
     unsigned external_haloed_node_index;
     external_haloed_node_index=
      mesh_pt->add_external_haloed_node_pt(iproc,nod_pt);

     // If it was added then the new index should match the size of the storage
     if (external_haloed_node_index==n_ext_haloed_nod)
      {
       // Indicate that this node needs to be constructed on
       // the other process
       send_unsigneds.push_back(1);
#ifdef ANNOTATE_MISSING_MASTERS_COMMUNICATION
       std::stringstream junk;
       junk << "Node needs to be constructed [size=" 
            << send_unsigneds.size() << "]; last entry: "
            << send_unsigneds[send_unsigneds.size()-1];
       Flat_packed_unsigneds_string.push_back(junk.str());
#endif

       // This helper function gets all the required information for the 
       // specified node and stores it into MPI-sendable information
       // so that a halo copy can be made on the receiving process
       get_required_nodal_information_helper(iproc,nod_pt,
                                             mesh_pt,
                                             n_cont_inter_values,
                                             send_unsigneds,
                                             send_doubles);
      }
     else // It was already added
      {
       // Indicate that this node doesn not need to be constructed on
       // the other process
       send_unsigneds.push_back(0); 
#ifdef ANNOTATE_MISSING_MASTERS_COMMUNICATION
       std::stringstream junk;
       junk << "Node was already added [size=" 
            << send_unsigneds.size() << "]; last entry: "
            << send_unsigneds[send_unsigneds.size()-1];

       Flat_packed_unsigneds_string.push_back(junk.str());
#endif

       // This node is already an external haloed node, so tell
       // the other process its index in the equivalent external halo storage
       send_unsigneds.push_back(0);
#ifdef ANNOTATE_MISSING_MASTERS_COMMUNICATION
       Flat_packed_unsigneds_string.push_back("haloed node found externally");
#endif
       send_unsigneds.push_back(external_haloed_node_index);
#ifdef ANNOTATE_MISSING_MASTERS_COMMUNICATION
       Flat_packed_unsigneds_string.push_back("external haloed node index");
#endif
      }
    }



  }


//==========start of add_external_haloed_master_node_helper===============
/// Helper function to add external haloed node that is a master
//========================================================================
 void add_external_haloed_master_node_helper(int& iproc, Node* master_nod_pt,
                                             Mesh* const &mesh_pt,
                                             int& n_cont_inter_values,
                                             Vector<unsigned>& send_unsigneds,
                                             Vector<double>& send_doubles)
  {
   //Check to see if this haloed node already exists in (internal)
   //haloed node storage with any processor
   bool found_internally = false;
   unsigned shared_node_index=0;

   //Get vector of all shared nodes with processor iproc
   Vector<Node*> shared_node_pt;
   mesh_pt->get_shared_node_pt(iproc,shared_node_pt);
 
   //Search the internal haloed storage for this node
   Vector<Node*>::iterator it
    = std::find(shared_node_pt.begin(),
                shared_node_pt.end(),
                master_nod_pt);
   
   //Check if the node was found in shared storage
   if(it != shared_node_pt.end())
    {
     //Node found in (internal) haloed storage
     found_internally = true;
     //Store the index in this storage
     shared_node_index = it - shared_node_pt.begin();
    }

   //// Slow search version without additional access function in Mesh class
   ////Search the internal shared node storage for this node
   //for(unsigned i=0; i<mesh_pt->nshared_node(iproc); i++)
   // {
   //  if(master_nod_pt == mesh_pt->shared_node_pt(iproc,i))
   //   {
   //    //Node found in (internal) shared storage
   //    found_internally = true;
   //    shared_node_index = i;
   //    break;
   //   }
   // }

   // If we've found the node internally
   if(found_internally)
    {
     // Indicate that this node doesn not need to be constructed on
     // the other process
     send_unsigneds.push_back(0);
#ifdef ANNOTATE_MISSING_MASTERS_COMMUNICATION
     std::stringstream junk;
     junk << "Node was already added [size=" 
          << send_unsigneds.size() << "]; last entry: "
          << send_unsigneds[send_unsigneds.size()-1];

     Flat_packed_unsigneds_string.push_back(junk.str());
#endif

     // This node is already shared with processor iproc, so tell the other
     // processor its index in the shared node storage
     send_unsigneds.push_back(1);
#ifdef ANNOTATE_MISSING_MASTERS_COMMUNICATION
     Flat_packed_unsigneds_string.push_back("haloed node found internally");
#endif
     send_unsigneds.push_back(shared_node_index);
#ifdef ANNOTATE_MISSING_MASTERS_COMMUNICATION
     Flat_packed_unsigneds_string.push_back("(internal) haloed node index");
#endif
    }
   else
    {
     // Attempt to add node as an external haloed node
     unsigned n_ext_haloed_nod=mesh_pt->nexternal_haloed_node(iproc);
     unsigned external_haloed_node_index;
     external_haloed_node_index=
      mesh_pt->add_external_haloed_node_pt(iproc,master_nod_pt);

     // If it was added the returned index is the same as current storage size
     if (external_haloed_node_index==n_ext_haloed_nod)
      {
       // Indicate that this node needs to be constructed on
       // the other process
       send_unsigneds.push_back(1);
#ifdef ANNOTATE_MISSING_MASTERS_COMMUNICATION
       Flat_packed_unsigneds_string.push_back("Node needs to be constructed[2]");
#endif

       // This gets all the required information for the specified
       // master node and stores it into MPI-sendable information
       // so that a halo copy can be made on the receiving process
       get_required_master_nodal_information_helper(iproc,master_nod_pt,
                                                    mesh_pt,
                                                    n_cont_inter_values,
                                                    send_unsigneds,
                                                    send_doubles);
      }
     else // It was already added
      {
       // Indicate that this node doesn not need to be constructed on
       // the other process
       send_unsigneds.push_back(0);
#ifdef ANNOTATE_MISSING_MASTERS_COMMUNICATION
       Flat_packed_unsigneds_string.push_back("Node was already added[2]");
#endif

       // This node is already an external haloed node, so tell
       // the other process its index in the equivalent external halo storage
       send_unsigneds.push_back(0);
#ifdef ANNOTATE_MISSING_MASTERS_COMMUNICATION
       Flat_packed_unsigneds_string.push_back("haloed node found externally");
#endif
       send_unsigneds.push_back(external_haloed_node_index);
#ifdef ANNOTATE_MISSING_MASTERS_COMMUNICATION
       Flat_packed_unsigneds_string.push_back("external haloed node index[2]");
#endif
      }
    }
  }




//========start of get_required_nodal_information_helper==================
/// Helper function to get the required nodal information from an
/// external haloed node so that a fully-functional external halo
/// node (and therefore element) can be created on the receiving process
//========================================================================
 void get_required_nodal_information_helper(int& iproc, Node* nod_pt,
                                            Mesh* const &mesh_pt,
                                            int& n_cont_inter_values,
                                            Vector<unsigned>& send_unsigneds,
                                            Vector<double>& send_doubles)
  {
   // Tell the halo copy of this node how many values there are
   // [NB this may be different for nodes within the same element, e.g.
   //  when using Lagrange multipliers]
   unsigned n_val=nod_pt->nvalue();
   send_unsigneds.push_back(n_val);
#ifdef ANNOTATE_MISSING_MASTERS_COMMUNICATION
   Flat_packed_unsigneds_string.push_back("Number of values");
#endif

   unsigned n_dim=nod_pt->ndim();
   TimeStepper* time_stepper_pt=nod_pt->time_stepper_pt();
   
   // Default number of previous values to 1
   unsigned n_prev=1;
   if (time_stepper_pt!=0)
    {
     // Add number of history values to n_prev
     n_prev=time_stepper_pt->ntstorage(); 
    }

   // Is the node on any boundaries?
   if (nod_pt->is_on_boundary())
    {
     send_unsigneds.push_back(1);
#ifdef ANNOTATE_MISSING_MASTERS_COMMUNICATION
     Flat_packed_unsigneds_string.push_back("Node is on boundary");
#endif

     // Loop over the boundaries of the external mesh
     Vector<unsigned> boundaries;
     unsigned n_bnd=mesh_pt->nboundary();
     for (unsigned i_bnd=0;i_bnd<n_bnd;i_bnd++)
      {
       // Which boundaries (could be more than one) is it on?
       if (nod_pt->is_on_boundary(i_bnd))
        {
         boundaries.push_back(i_bnd);
        }
      }
     unsigned nb=boundaries.size();
     send_unsigneds.push_back(nb);
#ifdef ANNOTATE_MISSING_MASTERS_COMMUNICATION
     std::stringstream junk;
     junk << "Node is on "<< nb << " boundaries";
     Flat_packed_unsigneds_string.push_back(junk.str());
#endif
     for (unsigned i=0;i<nb;i++)
      {
       send_unsigneds.push_back(boundaries[i]);
#ifdef ANNOTATE_MISSING_MASTERS_COMMUNICATION
       std::stringstream junk;
       junk << "Node is on boundary " << boundaries[i] << " of " << n_bnd;
       Flat_packed_unsigneds_string.push_back(junk.str());
#endif
      }

     // Get pointer to the map of indices associated with
     // additional values created by face elements
     BoundaryNodeBase* bnod_pt=
      dynamic_cast<BoundaryNodeBase*>(nod_pt);
#ifdef PARANOID
     if (bnod_pt==0)
      {
       throw OomphLibError(
        "Failed to cast new node to boundary node\n",
        OOMPH_CURRENT_FUNCTION,
        OOMPH_EXCEPTION_LOCATION);
      }
#endif
     std::map<unsigned, unsigned>* map_pt=
      bnod_pt->index_of_first_value_assigned_by_face_element_pt();
     
     // No additional values created
     if (map_pt==0)
      {
       send_unsigneds.push_back(0);
#ifdef ANNOTATE_MISSING_MASTERS_COMMUNICATION
       std::stringstream junk;
       Flat_packed_unsigneds_string.push_back("No additional values were created by face element");
#endif
      }
     // Created additional values
     else
      {
       // How many?
       send_unsigneds.push_back(map_pt->size());
#ifdef ANNOTATE_MISSING_MASTERS_COMMUNICATION
       std::stringstream junk;
       junk << "Map size " << map_pt->size() << n_bnd;
       Flat_packed_unsigneds_string.push_back(junk.str());
#endif
       // Loop over entries in map and add to send data
       for (std::map<unsigned, unsigned>::iterator p=
             map_pt->begin();
            p!=map_pt->end();p++)
        {
         send_unsigneds.push_back((*p).first);
#ifdef ANNOTATE_MISSING_MASTERS_COMMUNICATION
         std::stringstream junk;
         Flat_packed_unsigneds_string.push_back("Key of map entry");
#endif
         send_unsigneds.push_back((*p).second);
#ifdef ANNOTATE_MISSING_MASTERS_COMMUNICATION
         Flat_packed_unsigneds_string.push_back("Value of map entry");
#endif
        }
      }
    }
   else
    {
     // Not on any boundary
     send_unsigneds.push_back(0);
#ifdef ANNOTATE_MISSING_MASTERS_COMMUNICATION
     Flat_packed_unsigneds_string.push_back("Node is not on any boundary");
#endif
    }

   // Is the Node algebraic?  If so, send its ref values and
   // an indication of its geometric objects if they are stored
   // in the algebraic mesh
   AlgebraicNode* alg_nod_pt=dynamic_cast<AlgebraicNode*>(nod_pt);
   if (alg_nod_pt!=0)
    {
     // The external mesh should be algebraic
     AlgebraicMesh* alg_mesh_pt=dynamic_cast<AlgebraicMesh*>
      (mesh_pt);

     // Get default node update function ID
     unsigned update_id=alg_nod_pt->node_update_fct_id();
     send_unsigneds.push_back(update_id);
#ifdef ANNOTATE_MISSING_MASTERS_COMMUNICATION
     Flat_packed_unsigneds_string.push_back("Alg Node update id");
#endif

     // Get reference values at default...
     unsigned n_ref_val=alg_nod_pt->nref_value();
     send_unsigneds.push_back(n_ref_val);
#ifdef ANNOTATE_MISSING_MASTERS_COMMUNICATION
     Flat_packed_unsigneds_string.push_back("Alg Node n ref values");
#endif
     for (unsigned i_ref_val=0;i_ref_val<n_ref_val;i_ref_val++)
      {
       send_doubles.push_back(alg_nod_pt->ref_value(i_ref_val));
      }

     // Access geometric objects at default...
     unsigned n_geom_obj=alg_nod_pt->ngeom_object();
     send_unsigneds.push_back(n_geom_obj);
#ifdef ANNOTATE_MISSING_MASTERS_COMMUNICATION
     Flat_packed_unsigneds_string.push_back("Alg Node n geom objects");
#endif
     for (unsigned i_geom=0;i_geom<n_geom_obj;i_geom++)
      {
       GeomObject* geom_obj_pt=alg_nod_pt->geom_object_pt(i_geom);

       // Check this against the stored geometric objects in mesh
       unsigned n_geom_list=alg_mesh_pt->ngeom_object_list_pt();

       // Default found index to zero
       unsigned found_geom_object=0;
       for (unsigned i_list=0;i_list<n_geom_list;i_list++)
        {
         if (geom_obj_pt==alg_mesh_pt->geom_object_list_pt(i_list))
          {
           found_geom_object=i_list;
          }
        }
       send_unsigneds.push_back(found_geom_object);
#ifdef ANNOTATE_MISSING_MASTERS_COMMUNICATION
       Flat_packed_unsigneds_string.push_back("Found geom object");
#endif
      }
    }

   // If it is a MacroElementNodeUpdateNode, everything has been
   // dealt with by the new element already

   // Is it a SolidNode?
   SolidNode* solid_nod_pt=dynamic_cast<SolidNode*>(nod_pt);
   if (solid_nod_pt!=0)
    {
     unsigned n_solid_val=solid_nod_pt->variable_position_pt()->nvalue();
     for (unsigned i_val=0;i_val<n_solid_val;i_val++)
      {
       for (unsigned t=0;t<n_prev;t++)
        {
         send_doubles.push_back(solid_nod_pt->variable_position_pt()->
                          value(t,i_val));
        }
      }
    }

   // Finally copy info required for all node types
   for (unsigned i_val=0;i_val<n_val;i_val++)
    {
     for (unsigned t=0;t<n_prev;t++)
      {
       send_doubles.push_back(nod_pt->value(t,i_val));
      }
    }

   // Now do positions
   for (unsigned idim=0;idim<n_dim;idim++)
    {
     for (unsigned t=0;t<n_prev;t++)
      {
       send_doubles.push_back(nod_pt->x(t,idim));
      }
    }
  }

//=========start of get_required_master_nodal_information_helper==========
/// Helper function to get the required master nodal information from an
/// external haloed master node so that a fully-functional external halo
/// master node (and possible element) can be created on the receiving process
//========================================================================
 void get_required_master_nodal_information_helper
 (int& iproc, Node* master_nod_pt,
  Mesh* const &mesh_pt, int& n_cont_inter_values,
  Vector<unsigned>& send_unsigneds,
  Vector<double>& send_doubles)
  {
   // Need to send over dimension, position type and number of values
   send_unsigneds.push_back(master_nod_pt->ndim());
#ifdef ANNOTATE_MISSING_MASTERS_COMMUNICATION
   Flat_packed_unsigneds_string.push_back("Master node ndim");
#endif
   send_unsigneds.push_back(master_nod_pt->nposition_type());
#ifdef ANNOTATE_MISSING_MASTERS_COMMUNICATION
   Flat_packed_unsigneds_string.push_back("Master node npos_type");
#endif
   send_unsigneds.push_back(master_nod_pt->nvalue());
#ifdef ANNOTATE_MISSING_MASTERS_COMMUNICATION
   Flat_packed_unsigneds_string.push_back("Master node nvalue");
#endif
   if(master_nod_pt->is_halo())
    {
     send_unsigneds.push_back(master_nod_pt->non_halo_proc_ID());
    }
   else
    {
     send_unsigneds.push_back(mesh_pt->communicator_pt()->my_rank());
    }
#ifdef ANNOTATE_MISSING_MASTERS_COMMUNICATION
   Flat_packed_unsigneds_string.push_back("Master node non-halo processor ID");
#endif
   
   // If it's a solid node, also need to send lagrangian dim and type
   SolidNode* solid_nod_pt=dynamic_cast<SolidNode*>(master_nod_pt);
   if (solid_nod_pt!=0)
    {
     send_unsigneds.push_back(solid_nod_pt->nlagrangian());
#ifdef ANNOTATE_MISSING_MASTERS_COMMUNICATION
     Flat_packed_unsigneds_string.push_back("Master solid node nlagr");
#endif
     send_unsigneds.push_back(solid_nod_pt->nlagrangian_type());
#ifdef ANNOTATE_MISSING_MASTERS_COMMUNICATION
     Flat_packed_unsigneds_string.push_back("Master solid node nlagr_type");
#endif
    }

   unsigned n_dim=master_nod_pt->ndim();
   TimeStepper* time_stepper_pt=master_nod_pt->time_stepper_pt();
   
   // Default number of previous values to 1
   unsigned n_prev=1;
   if (time_stepper_pt!=0)
    {
     // Add number of history values to n_prev
     n_prev=time_stepper_pt->ntstorage();
    }

   // Is the node on any boundaries?
   if (master_nod_pt->is_on_boundary())
    {
     send_unsigneds.push_back(1);
#ifdef ANNOTATE_MISSING_MASTERS_COMMUNICATION
     Flat_packed_unsigneds_string.push_back("Master node is on boundary");
#endif
     // Loop over the boundaries of the external mesh
     Vector<unsigned> boundaries;
     unsigned n_bnd=mesh_pt->nboundary();
     for (unsigned i_bnd=0;i_bnd<n_bnd;i_bnd++)
      {
       // Which boundaries (could be more than one) is it on?
       if (master_nod_pt->is_on_boundary(i_bnd))
        {
         boundaries.push_back(i_bnd);
        }
      }
     unsigned nb=boundaries.size();
     send_unsigneds.push_back(nb);
#ifdef ANNOTATE_MISSING_MASTERS_COMMUNICATION
     std::stringstream junk;
     junk << "Master node is on "<< nb << " boundaries";
     Flat_packed_unsigneds_string.push_back(junk.str());
#endif
     for (unsigned i=0;i<nb;i++)
      {
       send_unsigneds.push_back(boundaries[i]);
#ifdef ANNOTATE_MISSING_MASTERS_COMMUNICATION
       std::stringstream junk;
       junk << "Master noode is on boundary "
            << boundaries[i] << " of " << n_bnd;
       Flat_packed_unsigneds_string.push_back(junk.str());
#endif
      }

     // Get pointer to the map of indices associated with
     // additional values created by face elements
     BoundaryNodeBase* bnod_pt=
      dynamic_cast<BoundaryNodeBase*>(master_nod_pt);
#ifdef PARANOID
     if (bnod_pt==0)
      {
       throw OomphLibError(
        "Failed to cast new node to boundary node\n",
        OOMPH_CURRENT_FUNCTION,
        OOMPH_EXCEPTION_LOCATION);
      }
#endif
     std::map<unsigned, unsigned>* map_pt=
      bnod_pt->index_of_first_value_assigned_by_face_element_pt();
     
     // No additional values created
     if (map_pt==0)
      {
       send_unsigneds.push_back(0);
#ifdef ANNOTATE_MISSING_MASTERS_COMMUNICATION
       std::stringstream junk;
       Flat_packed_unsigneds_string.push_back("No additional values were created by face element for this master node");
#endif
      }
     // Created additional values
     else
      {
       // How many?
       send_unsigneds.push_back(map_pt->size());
#ifdef ANNOTATE_MISSING_MASTERS_COMMUNICATION
       std::stringstream junk;
       junk << "Map size for master node " << map_pt->size() << n_bnd;
       Flat_packed_unsigneds_string.push_back(junk.str());
#endif
       // Loop over entries in map and add to send data
       for (std::map<unsigned, unsigned>::iterator p=
             map_pt->begin();
            p!=map_pt->end();p++)
        {
         send_unsigneds.push_back((*p).first);
#ifdef ANNOTATE_MISSING_MASTERS_COMMUNICATION
         std::stringstream junk;
         Flat_packed_unsigneds_string.push_back(
          "Key of map entry for master node");
#endif
         send_unsigneds.push_back((*p).second);
#ifdef ANNOTATE_MISSING_MASTERS_COMMUNICATION
         Flat_packed_unsigneds_string.push_back(
          "Value of map entry for master node");
#endif
        }
      }
    }
   else
    {
     // Not on any boundary
     send_unsigneds.push_back(0);
#ifdef ANNOTATE_MISSING_MASTERS_COMMUNICATION
     Flat_packed_unsigneds_string.push_back("Master node is not on any boundary");
#endif
    }

   // Is the Node algebraic?  If so, send its ref values and
   // an indication of its geometric objects if they are stored
   // in the algebraic mesh
   AlgebraicNode* alg_nod_pt=dynamic_cast<AlgebraicNode*>(master_nod_pt);
   if (alg_nod_pt!=0)
    {
     // The external mesh should be algebraic
     AlgebraicMesh* alg_mesh_pt=dynamic_cast<AlgebraicMesh*>
      (mesh_pt);

     // Get default node update function ID
     unsigned update_id=alg_nod_pt->node_update_fct_id();
     send_unsigneds.push_back(update_id);
#ifdef ANNOTATE_MISSING_MASTERS_COMMUNICATION
     Flat_packed_unsigneds_string.push_back("Master Alg Node update id");
#endif

     // Get reference values at default...
     unsigned n_ref_val=alg_nod_pt->nref_value();
     send_unsigneds.push_back(n_ref_val);
#ifdef ANNOTATE_MISSING_MASTERS_COMMUNICATION
     Flat_packed_unsigneds_string.push_back("Master Alg Node n ref values");
#endif
     for (unsigned i_ref_val=0;i_ref_val<n_ref_val;i_ref_val++)
      {
       send_doubles.push_back(alg_nod_pt->ref_value(i_ref_val));
      }

     // Access geometric objects at default...
     unsigned n_geom_obj=alg_nod_pt->ngeom_object();
     send_unsigneds.push_back(n_geom_obj);
#ifdef ANNOTATE_MISSING_MASTERS_COMMUNICATION
     Flat_packed_unsigneds_string.push_back("Master Alg Node n geom objects");
#endif
     for (unsigned i_geom=0;i_geom<n_geom_obj;i_geom++)
      {
       GeomObject* geom_obj_pt=alg_nod_pt->geom_object_pt(i_geom);
       // Check this against the stored geometric objects in mesh
       unsigned n_geom_list=alg_mesh_pt->ngeom_object_list_pt();
       // Default found index to zero
       unsigned found_geom_object=0;
       for (unsigned i_list=0;i_list<n_geom_list;i_list++)
        {
         if (geom_obj_pt==alg_mesh_pt->geom_object_list_pt(i_list))
          {
           found_geom_object=i_list;
          }
        }
       send_unsigneds.push_back(found_geom_object);
#ifdef ANNOTATE_MISSING_MASTERS_COMMUNICATION
       Flat_packed_unsigneds_string.push_back("Master node Found geom object");
#endif
      }
    } // end AlgebraicNode check

   // Is it a MacroElementNodeUpdateNode?
   MacroElementNodeUpdateNode* macro_nod_pt=
    dynamic_cast<MacroElementNodeUpdateNode*>(master_nod_pt);
   if (macro_nod_pt!=0)
    {
     oomph_info << "Adding external haloed master node: " << master_nod_pt << " at " << master_nod_pt->x(0) << ", " << master_nod_pt->x(1) << " ]" << std::endl;
     // Loop over current external haloed elements - has the element which
     // controls the node update for this node been added yet?
     GeneralisedElement* macro_node_update_el_pt=
      macro_nod_pt->node_update_element_pt();

     //BENFLAG: Check that the node's macro element node update element actually contains the node
     oomph_info << "Master node's macro update element:" << std::endl;
     bool really_bad = true;
     FiniteElement* mac_el_pt = dynamic_cast<FiniteElement*>(macro_node_update_el_pt);
     for(unsigned j=0; j<mac_el_pt->nnode(); j++)
      {
       oomph_info << mac_el_pt->node_pt(j) << ": [ " << mac_el_pt->node_pt(j)->x(0) << ", " << mac_el_pt->node_pt(j)->x(1) << " ] " << std::endl;
       if(mac_el_pt->node_pt(j)==master_nod_pt)
        {
         really_bad = false;
         //oomph_info << "Found it!" << std::endl;
        }
      }
     if(really_bad==true)
      {
       oomph_info << "This is REALLY BAD! The master node is not part of its own update element..." << std::endl;
      }

     //BENFLAG: Search internal storage for node update element
     Vector<GeneralisedElement*> int_haloed_el_pt(mesh_pt->haloed_element_pt(iproc));
     //unsigned n_int_haloed_el=int_haloed_el_pt.size();
     Vector<GeneralisedElement*>::iterator it = std::find(int_haloed_el_pt.begin(),int_haloed_el_pt.end(),macro_node_update_el_pt);
     if(it != int_haloed_el_pt.end())
      {
       // Found in internal haloed storage
       unsigned int_haloed_el_index = it - int_haloed_el_pt.begin();
       oomph_info << "Found internally at index " << int_haloed_el_index << std::endl;
       //BENFLAG: Check index corresponds to correct element
       if((mesh_pt->haloed_element_pt(iproc))[int_haloed_el_index]!=macro_node_update_el_pt)
        {
         oomph_info << "Found wrong index!!!" << std::endl;
         throw;
        }
       else
        {
         oomph_info << "index and proc are correct in internal storage." << std::endl;
         oomph_info << "i.e. " << (mesh_pt->haloed_element_pt(iproc))[int_haloed_el_index] << "==" << macro_node_update_el_pt << std::endl;
        }

       
     
       // Say haloed element already exists
       send_unsigneds.push_back(0);
#ifdef ANNOTATE_MISSING_MASTERS_COMMUNICATION
       Flat_packed_unsigneds_string.push_back("External haloed element already exists");
#endif
       // Say found internally
       send_unsigneds.push_back(1);
#ifdef ANNOTATE_MISSING_MASTERS_COMMUNICATION
       Flat_packed_unsigneds_string.push_back("Haloed element found internally");
#endif
       // Say what the index is
       send_unsigneds.push_back(int_haloed_el_index);
#ifdef ANNOTATE_MISSING_MASTERS_COMMUNICATION
       Flat_packed_unsigneds_string.push_back("Index of existing internal haloed element");
#endif
       //BENFLAG:
       FiniteElement* tmp_el_pt = dynamic_cast<FiniteElement*>((mesh_pt->haloed_element_pt(iproc))[int_haloed_el_index]);
       oomph_info << "Internal haloed element (" << tmp_el_pt << ") already exists..." << std::endl;
       oomph_info << "on proc " << iproc << " at index " << int_haloed_el_index << std::endl;
       for(unsigned j=0; j<tmp_el_pt->nnode(); j++)
        {
         oomph_info << tmp_el_pt->node_pt(j) << "==" << dynamic_cast<FiniteElement*>((mesh_pt->haloed_element_pt(iproc))[int_haloed_el_index])->node_pt(j) << " at [ " << tmp_el_pt->node_pt(j)->x(0) << ", " << tmp_el_pt->node_pt(j)->x(1) << " ]" << std::endl;
        }

       //oomph_info << "now " << tmp_el_pt << "==" << (mesh_pt->haloed_element_pt(iproc))[int_haloed_el_index] << "==" << macro_node_update_el_pt << std::endl;

//       //BENFLAG: Add to external storage too
//       unsigned n_ext_haloed_el=mesh_pt->
//        nexternal_haloed_element(iproc);
//       unsigned external_haloed_el_index;
//       external_haloed_el_index=mesh_pt->
//        add_external_haloed_element_pt(iproc,macro_node_update_el_pt);
//
//       // If it was already added, say
//       if (external_haloed_el_index!=n_ext_haloed_el)
//        {
//         oomph_info << "Element (" << tmp_el_pt << "==" << macro_node_update_el_pt << "==" << mesh_pt->external_haloed_element_pt(iproc,external_haloed_el_index) << ") also exists in external storage with proc " << iproc << " at index " << external_haloed_el_index << " of " << n_ext_haloed_el << "..." << std::endl;
//         // Say also exists in external storage
//         send_unsigneds.push_back(1234);
//#ifdef ANNOTATE_MISSING_MASTERS_COMMUNICATION
//         Flat_packed_unsigneds_string.push_back("Haloed element also found externally");
//#endif
//         // Say what the index is
//         send_unsigneds.push_back(external_haloed_el_index);
//#ifdef ANNOTATE_MISSING_MASTERS_COMMUNICATION
//         Flat_packed_unsigneds_string.push_back("Index of existing internal haloed element");
//#endif
//         oomph_info << "sent external_haloed_el_index = " << external_haloed_el_index << std::endl;
//         
//        }
//       else
//        {
//         oomph_info << "Element didn't exist in external storage with proc " << iproc << "..." << std::endl;
//         // Say doesn't exists in external storage
//         send_unsigneds.push_back(0);
//#ifdef ANNOTATE_MISSING_MASTERS_COMMUNICATION
//         Flat_packed_unsigneds_string.push_back("Haloed element not also found externally");
//#endif
//        }
      }
     else
      {

     unsigned n_ext_haloed_el=mesh_pt->
      nexternal_haloed_element(iproc);
     unsigned external_haloed_el_index;
     external_haloed_el_index=mesh_pt->
      add_external_haloed_element_pt(iproc,macro_node_update_el_pt);

     // If it wasn't already added, we need to create a halo copy
     if (external_haloed_el_index==n_ext_haloed_el)
      {
       send_unsigneds.push_back(1);
#ifdef ANNOTATE_MISSING_MASTERS_COMMUNICATION
       Flat_packed_unsigneds_string.push_back("Master Node needs to be constructed");
#endif
       //Cast to a finite elemnet
       FiniteElement* macro_node_update_finite_el_pt 
        = dynamic_cast<FiniteElement*>(macro_node_update_el_pt);
       
       // We're using macro elements to update...
       MacroElementNodeUpdateMesh* macro_mesh_pt=
        dynamic_cast<MacroElementNodeUpdateMesh*>(mesh_pt);
       if (macro_mesh_pt!=0)
        {
         send_unsigneds.push_back(1);
#ifdef ANNOTATE_MISSING_MASTERS_COMMUNICATION
         Flat_packed_unsigneds_string.push_back("Mesh is macro element mesh");
#endif
         // Need to send the macro element number in the mesh across
         MacroElement* macro_el_pt= macro_node_update_finite_el_pt
          ->macro_elem_pt();
         unsigned macro_el_num=macro_el_pt->macro_element_number();
         send_unsigneds.push_back(macro_el_num);
#ifdef ANNOTATE_MISSING_MASTERS_COMMUNICATION
         Flat_packed_unsigneds_string.push_back("Number of macro element");
#endif
         // Also need to send
         // the lower left and upper right coordinates of the macro element
         QElementBase* q_el_pt=dynamic_cast<QElementBase*>
          (macro_node_update_el_pt);
         if (q_el_pt!=0)
          {
           // The macro element needs to be set first before
           // its lower left and upper right coordinates can be accessed
           // Now send the lower left and upper right coordinates
           unsigned el_dim=q_el_pt->dim();
           for (unsigned i_dim=0;i_dim<el_dim;i_dim++)
            {
             send_doubles.push_back(q_el_pt->s_macro_ll(i_dim));
             send_doubles.push_back(q_el_pt->s_macro_ur(i_dim));
            }
          }
         else // Throw an error
          {
           std::ostringstream error_stream;
           error_stream << "You are using a MacroElement node update\n"
                        << "in a case with non-QElements. This has not\n"
                        << "yet been implemented.\n";
           throw OomphLibError
            (error_stream.str(),
             OOMPH_CURRENT_FUNCTION,
             OOMPH_EXCEPTION_LOCATION);
          }

        }
       else // Not using macro elements for node update... umm, we're
            // already inside a loop over macro elements, so this
            // should never get here... an error should be thrown I suppose
        {
         send_unsigneds.push_back(0);
#ifdef ANNOTATE_MISSING_MASTERS_COMMUNICATION
         Flat_packed_unsigneds_string.push_back("Mesh is not a macro element mesh");
#endif
        }

       // If the element is p-refineable we need to send the p-order so that the halo
       // version can be constructed correctly
       PRefineableElement* p_refineable_el_pt =
        dynamic_cast<PRefineableElement*>(macro_node_update_finite_el_pt);
       if(p_refineable_el_pt!=0)
        {
         send_unsigneds.push_back(1);
#ifdef ANNOTATE_MISSING_MASTERS_COMMUNICATION
         Flat_packed_unsigneds_string.push_back("Element is p-refineable");
#endif
         // Send p-order of macro element node update element
         unsigned p_order = p_refineable_el_pt->p_order();
         send_unsigneds.push_back(p_order);
#ifdef ANNOTATE_MISSING_MASTERS_COMMUNICATION
         Flat_packed_unsigneds_string.push_back("p-order of element");
#endif
        }
       else
        {
         send_unsigneds.push_back(0);
#ifdef ANNOTATE_MISSING_MASTERS_COMMUNICATION
         Flat_packed_unsigneds_string.push_back("Element is not p-refineable");
#endif
        }

       // This element needs to be fully functioning on the other
       // process, so send all the information required to create it
       unsigned n_node=macro_node_update_finite_el_pt->nnode();
       for (unsigned j=0;j<n_node;j++)
        {
         Node* new_nod_pt=macro_node_update_finite_el_pt->node_pt(j);
         add_external_haloed_node_to_storage(iproc,new_nod_pt,
                                             mesh_pt,
                                             n_cont_inter_values,
                                             send_unsigneds,
                                             send_doubles);
        }
      }
     else // The external haloed element already exists
      {
       // Say haloed element already exists
       send_unsigneds.push_back(0);
#ifdef ANNOTATE_MISSING_MASTERS_COMMUNICATION
       Flat_packed_unsigneds_string.push_back("External haloed element already exists");
#endif
       // Say found externally
       send_unsigneds.push_back(0);
#ifdef ANNOTATE_MISSING_MASTERS_COMMUNICATION
       Flat_packed_unsigneds_string.push_back("Haloed element found externally");
#endif
       send_unsigneds.push_back(external_haloed_el_index);
#ifdef ANNOTATE_MISSING_MASTERS_COMMUNICATION
       Flat_packed_unsigneds_string.push_back("Index of existing external haloed element");
#endif
       //BENFLAG:
       oomph_info << "External haloed element already exists..." << std::endl;
       FiniteElement* tmp_el_pt = dynamic_cast<FiniteElement*>(mesh_pt->external_haloed_element_pt(iproc,external_haloed_el_index));
       oomph_info << "on proc " << iproc << " at index " << external_haloed_el_index << std::endl;
       for(unsigned j=0; j<tmp_el_pt->nnode(); j++)
        {
         oomph_info << tmp_el_pt->node_pt(j) << " at [ " << tmp_el_pt->node_pt(j)->x(0) << ", " << tmp_el_pt->node_pt(j)->x(1) << " ]" << std::endl;
        }
      }
      } // End of case where not found internally

    } // end of MacroElementNodeUpdateNode check 

   // Is it a SolidNode?
   if (solid_nod_pt!=0)
    {
     unsigned n_val=solid_nod_pt->variable_position_pt()->nvalue();
     for (unsigned i_val=0;i_val<n_val;i_val++)
      {
       for (unsigned t=0;t<n_prev;t++)
        {
         send_doubles.push_back(solid_nod_pt->variable_position_pt()->
                          value(t,i_val));
        }
      }
    }

   // Finally copy info required for all node types

   // Halo copy needs to know all the history values
   unsigned n_val=master_nod_pt->nvalue();
   for (unsigned i_val=0;i_val<n_val;i_val++)
    {
     for (unsigned t=0;t<n_prev;t++)
      {
       send_doubles.push_back(master_nod_pt->value(t,i_val));
      }
    }

   // Now do positions
   for (unsigned idim=0;idim<n_dim;idim++)
    {
     for (unsigned t=0;t<n_prev;t++)
      {
       send_doubles.push_back(master_nod_pt->x(t,idim));
      }
    }

  }







//=======start of add_external_halo_node_helper===========================
/// Helper functiono to add external halo node that is not a master
//========================================================================
  void add_external_halo_node_helper
  (Node* &new_nod_pt, Mesh* const &mesh_pt, unsigned& loc_p,
   unsigned& node_index, FiniteElement* const &new_el_pt, 
   int& n_cont_inter_values,
   unsigned &counter_for_recv_unsigneds,
   Vector<unsigned>& recv_unsigneds,
   unsigned &counter_for_recv_doubles,
   Vector<double>& recv_doubles)
 {
  // Given the node and the external mesh, and received information
  // about them from process loc_p, construct them on the current process
#ifdef ANNOTATE_MISSING_MASTERS_COMMUNICATION
  oomph_info << "Rec:" << counter_for_recv_unsigneds 
             << " Bool: New node needs to be constructed " 
             << recv_unsigneds[counter_for_recv_unsigneds]
             << std::endl;
#endif
  if (recv_unsigneds[counter_for_recv_unsigneds++]==1)
   {
    // Construct a new node based upon sent information
    construct_new_external_halo_node_helper(new_nod_pt,loc_p,
                                            node_index,new_el_pt,
                                            mesh_pt,
                                            counter_for_recv_unsigneds,
                                            recv_unsigneds,
                                            counter_for_recv_doubles,
                                            recv_doubles);
   }
  else
   {
    // Need to check which storage we should copy this halo node from
#ifdef ANNOTATE_MISSING_MASTERS_COMMUNICATION
    oomph_info << "Rec:" << counter_for_recv_unsigneds 
               << "  Existing external halo node was found externally (0) or internally (1): " 
               << recv_unsigneds[counter_for_recv_unsigneds]
               << std::endl;
#endif
    unsigned node_found_internally
     = recv_unsigneds[counter_for_recv_unsigneds++];
    if(node_found_internally)
     {
#ifdef ANNOTATE_MISSING_MASTERS_COMMUNICATION
    oomph_info 
     << "Rec:" << counter_for_recv_unsigneds 
     << "  index of existing (internal) halo master node "
     << recv_unsigneds[counter_for_recv_unsigneds]
     << std::endl;
#endif

    // Copy node from received location
    new_nod_pt=mesh_pt->shared_node_pt
     (loc_p,recv_unsigneds[counter_for_recv_unsigneds++]);

    new_el_pt->node_pt(node_index)=new_nod_pt;
     }
    else
     {
#ifdef ANNOTATE_MISSING_MASTERS_COMMUNICATION
    oomph_info << "Rec:" << counter_for_recv_unsigneds 
               << "  Index of existing external halo node " 
               << recv_unsigneds[counter_for_recv_unsigneds]
               << std::endl;
#endif

    // Copy node from received location
    new_nod_pt=mesh_pt->external_halo_node_pt
     (loc_p,recv_unsigneds[counter_for_recv_unsigneds++]);

    new_el_pt->node_pt(node_index)=new_nod_pt;
     }


   }
 }




//========start of construct_new_external_halo_node_helper=================
/// Helper function which constructs a new external halo node (on new element)
/// with the required information sent from the haloed process
//========================================================================
 void construct_new_external_halo_node_helper
 (Node* &new_nod_pt, unsigned& loc_p, unsigned& node_index,
  FiniteElement* const &new_el_pt, 
  Mesh* const &mesh_pt,
  unsigned &counter_for_recv_unsigneds,
  Vector<unsigned>& recv_unsigneds,
  unsigned &counter_for_recv_doubles,
  Vector<double>& recv_doubles)
 {
  // The first entry indicates the number of values at this new Node
  // (which may be different across the same element e.g. Lagrange multipliers)
#ifdef ANNOTATE_MISSING_MASTERS_COMMUNICATION
  oomph_info << "Rec:" << counter_for_recv_unsigneds 
             << "  Number of values of external halo node " 
             << recv_unsigneds[counter_for_recv_unsigneds]
             << std::endl;
#endif
  unsigned n_val=recv_unsigneds[counter_for_recv_unsigneds++];

  // Null TimeStepper for now
  TimeStepper* time_stepper_pt=0;
  // Default number of previous values to 1
  unsigned n_prev=1;

  //Just take timestepper from a node
  //Let's use first node of first element since this must exist
  time_stepper_pt=mesh_pt->
   finite_element_pt(0)->node_pt(0)->time_stepper_pt();

  // If this node was on a boundary then it needs to
  // be on the same boundary here
#ifdef ANNOTATE_MISSING_MASTERS_COMMUNICATION
  oomph_info << "Rec:" << counter_for_recv_unsigneds 
             << "  Is node on boundary? " 
             << recv_unsigneds[counter_for_recv_unsigneds]
             << std::endl;
#endif
  if (recv_unsigneds[counter_for_recv_unsigneds++]==1)
   {
    // Construct a new boundary node
    if (time_stepper_pt!=0)
     {
      new_nod_pt=new_el_pt->construct_boundary_node
       (node_index,time_stepper_pt);
     }
    else
     {
      new_nod_pt=new_el_pt->construct_boundary_node(node_index);
     }
    
    // How many boundaries does the node live on?
#ifdef ANNOTATE_MISSING_MASTERS_COMMUNICATION
    oomph_info << "Rec:" << counter_for_recv_unsigneds 
               << " Number of boundaries the node is on: " 
               << recv_unsigneds[counter_for_recv_unsigneds]
               << std::endl;
#endif
    unsigned nb=recv_unsigneds[counter_for_recv_unsigneds++];
    for (unsigned i=0;i<nb;i++)
     {
      // Boundary number
#ifdef ANNOTATE_MISSING_MASTERS_COMMUNICATION
      oomph_info << "Rec:" << counter_for_recv_unsigneds 
                 << "  Node is on boundary " 
                 << recv_unsigneds[counter_for_recv_unsigneds]
                 << std::endl;
#endif
      unsigned i_bnd=
       recv_unsigneds[counter_for_recv_unsigneds++];
      mesh_pt->add_boundary_node(i_bnd,new_nod_pt);
     }

    // Do we have additional values created by face elements?
#ifdef ANNOTATE_MISSING_MASTERS_COMMUNICATION
    oomph_info << "Rec:" << counter_for_recv_unsigneds 
               << " Number of additional values created by face element " 
               << recv_unsigneds[counter_for_recv_unsigneds]
               << std::endl;
#endif
    unsigned n_entry=recv_unsigneds[counter_for_recv_unsigneds++];
    if (n_entry>0)
     {
      // Create storage, if it doesn't already exist, for the map 
      // that will contain the position of the first entry of 
      // this face element's additional values, 
      BoundaryNodeBase* bnew_nod_pt=
       dynamic_cast<BoundaryNodeBase*>(new_nod_pt);
#ifdef PARANOID
      if (bnew_nod_pt==0)
       {
        throw OomphLibError(
         "Failed to cast new node to boundary node\n",
         OOMPH_CURRENT_FUNCTION,
         OOMPH_EXCEPTION_LOCATION);
       }
#endif
      if(bnew_nod_pt->
         index_of_first_value_assigned_by_face_element_pt()==0)
       {
        bnew_nod_pt->
         index_of_first_value_assigned_by_face_element_pt()= 
         new std::map<unsigned, unsigned>; 
       }
      
      // Get pointer to the map of indices associated with
      // additional values created by face elements
      std::map<unsigned, unsigned>* map_pt=
       bnew_nod_pt->index_of_first_value_assigned_by_face_element_pt();
      
      // Loop over number of entries in map
      for (unsigned i=0;i<n_entry;i++)
       {
        // Read out pairs...

#ifdef ANNOTATE_MISSING_MASTERS_COMMUNICATION
        oomph_info << "Rec:" << counter_for_recv_unsigneds 
                   << " Key of map entry" 
                   << recv_unsigneds[counter_for_recv_unsigneds]
                   << std::endl;
#endif
        unsigned first=recv_unsigneds[counter_for_recv_unsigneds++];

#ifdef ANNOTATE_MISSING_MASTERS_COMMUNICATION
        oomph_info << "Rec:" << counter_for_recv_unsigneds 
                   << " Value of map entry" 
                   << recv_unsigneds[counter_for_recv_unsigneds]
                   << std::endl;
#endif
        unsigned second=recv_unsigneds[counter_for_recv_unsigneds++];
        
        // ...and assign
        (*map_pt)[first]=second;
       }
     }
   }
  else
   {
    // Construct an ordinary (non-boundary) node
    if (time_stepper_pt!=0)
     {
      new_nod_pt=new_el_pt->construct_node
       (node_index,time_stepper_pt);
     }
    else
     {
      new_nod_pt=new_el_pt->construct_node(node_index);
     }
   }

  // Node constructed: add to external halo nodes
  mesh_pt->add_external_halo_node_pt(loc_p,new_nod_pt);

  // Is the new constructed node Algebraic?
  AlgebraicNode* new_alg_nod_pt=dynamic_cast<AlgebraicNode*>
   (new_nod_pt);

  // If it is algebraic, its node update functions will
  // not yet have been set up properly
  if (new_alg_nod_pt!=0)
   {
    // The AlgebraicMesh is the external mesh
    AlgebraicMesh* alg_mesh_pt=dynamic_cast<AlgebraicMesh*>
     (mesh_pt);

    /// The first entry of All_alg_nodal_info contains
    /// the default node update id
    /// e.g. for the quarter circle there are 
    /// "Upper_left_box", "Lower right box" etc...
#ifdef ANNOTATE_MISSING_MASTERS_COMMUNICATION
    oomph_info << "Rec:" << counter_for_recv_unsigneds 
               << "  Alg node update id "
               << recv_unsigneds[counter_for_recv_unsigneds]
               << std::endl;
#endif

    unsigned update_id=recv_unsigneds
     [counter_for_recv_unsigneds++];

    Vector<double> ref_value;

    // The size of this vector is in the next entry
    // of All_alg_nodal_info
#ifdef ANNOTATE_MISSING_MASTERS_COMMUNICATION
    oomph_info << "Rec:" << counter_for_recv_unsigneds 
               << "  Alg node # of ref values "
               << recv_unsigneds[counter_for_recv_unsigneds]
               << std::endl;
#endif
    unsigned n_ref_val=recv_unsigneds
     [counter_for_recv_unsigneds++];

    // The reference values themselves are in
    // All_alg_ref_value
    ref_value.resize(n_ref_val);
    for (unsigned i_ref=0;i_ref<n_ref_val;i_ref++)
     {
      ref_value[i_ref]=recv_doubles
       [counter_for_recv_doubles++];
     }

    Vector<GeomObject*> geom_object_pt;
    /// again we need the size of this vector as it varies
    /// between meshes; we also need some indication
    /// as to which geometric object should be used...

    // The size of this vector is in the next entry
    // of All_alg_nodal_info
#ifdef ANNOTATE_MISSING_MASTERS_COMMUNICATION    
    oomph_info << "Rec:" << counter_for_recv_unsigneds 
               << "  Alg node # of geom objects "
               << recv_unsigneds[counter_for_recv_unsigneds]
               << std::endl;
#endif
    unsigned n_geom_obj=recv_unsigneds
     [counter_for_recv_unsigneds++];

    // The remaining indices are in the rest of 
    // All_alg_nodal_info
    geom_object_pt.resize(n_geom_obj);
    for (unsigned i_geom=0;i_geom<n_geom_obj;i_geom++)
     {
#ifdef ANNOTATE_MISSING_MASTERS_COMMUNICATION
      oomph_info << "Rec:" << counter_for_recv_unsigneds 
                 << "  Alg node: geom object index "
                 << recv_unsigneds[counter_for_recv_unsigneds]
                 << std::endl;
#endif
      unsigned geom_index=recv_unsigneds
       [counter_for_recv_unsigneds++];
      // This index indicates which of the AlgebraicMesh's
      // stored geometric objects should be used
      // (0 is a null pointer; everything else should have
      //  been filled in by the specific Mesh).  If it
      // hasn't been filled in then the update_node_update
      // call should fix it
      geom_object_pt[i_geom]=alg_mesh_pt->
       geom_object_list_pt(geom_index);
     }

    /// For the received update_id, ref_value, geom_object
    /// call add_node_update_info
    new_alg_nod_pt->add_node_update_info
     (update_id,alg_mesh_pt,geom_object_pt,ref_value);

    /// Now call update_node_update
    alg_mesh_pt->update_node_update(new_alg_nod_pt);
   }

  // Is the node a MacroElementNodeUpdateNode?
  MacroElementNodeUpdateNode* macro_nod_pt=
   dynamic_cast<MacroElementNodeUpdateNode*>(new_nod_pt);

  if (macro_nod_pt!=0)
   {
    // Need to call set_node_update_info; this requires
    // a Vector<GeomObject*> (taken from the mesh)
    Vector<GeomObject*> geom_object_vector_pt;

    // Access the required geom objects from the
    // MacroElementNodeUpdateMesh
    MacroElementNodeUpdateMesh* macro_mesh_pt=
     dynamic_cast<MacroElementNodeUpdateMesh*>
     (mesh_pt);
    geom_object_vector_pt=
     macro_mesh_pt->geom_object_vector_pt();

    // Get local coordinate of node in new element
    Vector<double> s_in_macro_node_update_element;
    new_el_pt->local_coordinate_of_node
     (node_index,s_in_macro_node_update_element);

    // Set node update info for this node
    macro_nod_pt->set_node_update_info
     (new_el_pt,s_in_macro_node_update_element,
      geom_object_vector_pt);
   }

  // Is the new node a SolidNode? 
  SolidNode* solid_nod_pt=dynamic_cast<SolidNode*>(new_nod_pt);
  if (solid_nod_pt!=0)
   {
    unsigned n_solid_val=solid_nod_pt->variable_position_pt()->nvalue();
    for (unsigned i_val=0;i_val<n_solid_val;i_val++)
     {
      for (unsigned t=0;t<n_prev;t++)
       {
        solid_nod_pt->variable_position_pt()->
         set_value(t,i_val,
                   recv_doubles[counter_for_recv_doubles++]);
       }
     }
   }

  // If there are additional values, resize the node
  unsigned n_new_val=new_nod_pt->nvalue();
  if (n_val>n_new_val)
   {
    new_nod_pt->resize(n_val);
   }

  // Get copied history values
  //  unsigned n_val=new_nod_pt->nvalue();
  for (unsigned i_val=0;i_val<n_val;i_val++)
   {
    for (unsigned t=0;t<n_prev;t++)
     {
      new_nod_pt->set_value(t,i_val,recv_doubles
                            [counter_for_recv_doubles++]);
     }
   }

  // Get copied history values for positions
  unsigned n_dim=new_nod_pt->ndim();
  for (unsigned idim=0;idim<n_dim;idim++)
   {
    for (unsigned t=0;t<n_prev;t++)
     {
      // Copy to coordinate
      new_nod_pt->x(t,idim)=recv_doubles
       [counter_for_recv_doubles++];
     }
   }
 }

 
#endif

}

}