missing_masters.template.cc

Go to the documentation of this file.

//LIC// ====================================================================
//LIC// This file forms part of oomph-lib, the object-oriented, 
//LIC// multi-physics finite-element library, available 
//LIC// at http://www.oomph-lib.org.
//LIC// 
//LIC//    Version 1.0; svn revision $LastChangedRevision$
//LIC//
//LIC// $LastChangedDate$
//LIC// 
//LIC// Copyright (C) 2006-2016 Matthias Heil and Andrew Hazel
//LIC// 
//LIC// This library is free software; you can redistribute it and/or
//LIC// modify it under the terms of the GNU Lesser General Public
//LIC// License as published by the Free Software Foundation; either
//LIC// version 2.1 of the License, or (at your option) any later version.
//LIC// 
//LIC// This library is distributed in the hope that it will be useful,
//LIC// but WITHOUT ANY WARRANTY; without even the implied warranty of
//LIC// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
//LIC// Lesser General Public License for more details.
//LIC// 
//LIC// You should have received a copy of the GNU Lesser General Public
//LIC// License along with this library; if not, write to the Free Software
//LIC// Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
//LIC// 02110-1301  USA.
//LIC// 
//LIC// The authors may be contacted at oomph-lib@maths.man.ac.uk.
//LIC// 
//LIC//====================================================================
//Templated missing masters functions

//Include guards to prevent multiple inclusion of the header
#ifndef OOMPH_MISSING_MASTERS_CC
#define OOMPH_MISSING_MASTERS_CC

// Config header generated by autoconfig
#ifdef HAVE_CONFIG_H
#include <oomph-lib-config.h>
#endif

//Oomph-lib headers
#include "geom_objects.h"
#include "problem.h"
#include "shape.h"

#include "mesh.h"
#include "mesh_as_geometric_object.h"
#include "algebraic_elements.h"
#include "macro_element_node_update_element.h"
#include "Qelements.h"
#include "element_with_external_element.h"
#include "missing_masters.h"

namespace oomph
{

//// Templated helper functions for missing master methods,

#ifdef OOMPH_HAS_MPI


//============start of add_external_halo_node_to_storage===============
/// Helper function to add external halo nodes, including any masters,
/// based on information received from the haloed process
//=========================================================================
 template<class EXT_ELEMENT>
  void Missing_masters_functions::add_external_halo_node_to_storage
  (Node* &new_nod_pt, Mesh* const &mesh_pt, unsigned& loc_p,
   unsigned& node_index, FiniteElement* const &new_el_pt, 
   int& n_cont_inter_values,
   unsigned &counter_for_recv_unsigneds,
   Vector<unsigned>& recv_unsigneds,
   unsigned &counter_for_recv_doubles,
   Vector<double>& recv_doubles)
  {
   // Add the external halo node if required
   add_external_halo_node_helper(new_nod_pt,mesh_pt,loc_p,
                                 node_index,new_el_pt,
                                 n_cont_inter_values,
                                 counter_for_recv_unsigneds,
                                 recv_unsigneds,
                                 counter_for_recv_doubles,
                                 recv_doubles);
   
   // Recursively add masters
   recursively_add_masters_of_external_halo_node_to_storage<EXT_ELEMENT>
    (new_nod_pt, mesh_pt, loc_p,
     node_index, 
     n_cont_inter_values,
     counter_for_recv_unsigneds,
     recv_unsigneds,
     counter_for_recv_doubles,
     recv_doubles);
   
   
  }
 
  
//========================================================================
/// Recursively add masters of external halo nodes (and their masters, etc)
/// based on information received from the haloed process
//=========================================================================
 template<class EXT_ELEMENT>
 void Missing_masters_functions::
 recursively_add_masters_of_external_halo_node_to_storage
 (Node* &new_nod_pt, Mesh* const &mesh_pt, unsigned& loc_p,
  unsigned& node_index,
  int& n_cont_inter_values,
  unsigned &counter_for_recv_unsigneds,
  Vector<unsigned>& recv_unsigneds,
  unsigned &counter_for_recv_doubles,
  Vector<double>& recv_doubles)
 {
  
  for (int i_cont=-1;i_cont<n_cont_inter_values;i_cont++)
   {
#ifdef ANNOTATE_MISSING_MASTERS_COMMUNICATION
    oomph_info 
     << "Rec:" << counter_for_recv_unsigneds 
     << " Boolean to indicate that continuously interpolated variable i_cont "
     << i_cont << " is hanging " 
     << recv_unsigneds[counter_for_recv_unsigneds]
     << std::endl;
#endif
    if (recv_unsigneds[counter_for_recv_unsigneds++]==1)
     {
#ifdef ANNOTATE_MISSING_MASTERS_COMMUNICATION
      oomph_info 
       << "Rec:" << counter_for_recv_unsigneds 
       << "  Number of master nodes "
       << recv_unsigneds[counter_for_recv_unsigneds]
       << std::endl;
#endif
      unsigned n_master=recv_unsigneds
       [counter_for_recv_unsigneds++];
      
      // Setup new HangInfo
      HangInfo* hang_pt=new HangInfo(n_master);
      for (unsigned m=0;m<n_master;m++)
       {
        Node* master_nod_pt=0;
        // Get the master node (creating and adding it if required)
        add_external_halo_master_node_helper<EXT_ELEMENT>
         (master_nod_pt,new_nod_pt,mesh_pt,loc_p,
          n_cont_inter_values,
          counter_for_recv_unsigneds,
          recv_unsigneds,
          counter_for_recv_doubles,
          recv_doubles);
        
        // Get the weight and set the HangInfo
        double master_weight=recv_doubles
         [counter_for_recv_doubles++];
        hang_pt->set_master_node_pt(m,master_nod_pt,master_weight);

        // Recursively add masters of master
        recursively_add_masters_of_external_halo_node_to_storage<EXT_ELEMENT>
         (master_nod_pt, mesh_pt, loc_p,
          node_index, 
          n_cont_inter_values,
          counter_for_recv_unsigneds,
          recv_unsigneds,
          counter_for_recv_doubles,
          recv_doubles);
       }
      new_nod_pt->set_hanging_pt(hang_pt,i_cont);
     }
   } // end loop over continous interpolated values
  
 }
 
//========================================================================
/// Helper function to add external halo node that is a master
//========================================================================
template<class EXT_ELEMENT>
 void Missing_masters_functions::add_external_halo_master_node_helper
  (Node* &new_master_nod_pt, Node* &new_nod_pt, Mesh* const &mesh_pt,
   unsigned& loc_p, int& ncont_inter_values,
   unsigned &counter_for_recv_unsigneds,
   Vector<unsigned>& recv_unsigneds,
   unsigned &counter_for_recv_doubles,
   Vector<double>& recv_doubles)
 {
  // Given the node and the external mesh, and received information
  // about them from process loc_p, construct them on the current process
#ifdef ANNOTATE_MISSING_MASTERS_COMMUNICATION
  oomph_info 
   << "Rec:" << counter_for_recv_unsigneds 
   << "  Boolean to trigger construction of new external halo master node "
   << recv_unsigneds[counter_for_recv_unsigneds]
   << std::endl;
#endif
  if (recv_unsigneds[counter_for_recv_unsigneds++]==1)
   {
    // Construct a new node based upon sent information
    construct_new_external_halo_master_node_helper<EXT_ELEMENT>
     (new_master_nod_pt,new_nod_pt,loc_p,mesh_pt,
      counter_for_recv_unsigneds,
      recv_unsigneds,
      counter_for_recv_doubles,
      recv_doubles);
   }
  else
   {
    // Need to check which storage we should copy this halo node from
#ifdef ANNOTATE_MISSING_MASTERS_COMMUNICATION
    oomph_info << "Rec:" << counter_for_recv_unsigneds 
               << "  Existing external halo node was found externally (0) or internally (1): " 
               << recv_unsigneds[counter_for_recv_unsigneds]
               << std::endl;
#endif
    unsigned node_found_internally
     = recv_unsigneds[counter_for_recv_unsigneds++];
    if(node_found_internally)
     {
#ifdef ANNOTATE_MISSING_MASTERS_COMMUNICATION
    oomph_info 
     << "Rec:" << counter_for_recv_unsigneds 
     << "  index of existing (internal) halo master node "
     << recv_unsigneds[counter_for_recv_unsigneds]
     << std::endl;
#endif
    // Copy node from received location
    new_master_nod_pt=mesh_pt->shared_node_pt
     (loc_p,recv_unsigneds[counter_for_recv_unsigneds++]);
     }
    else
     {
#ifdef ANNOTATE_MISSING_MASTERS_COMMUNICATION
    oomph_info 
     << "Rec:" << counter_for_recv_unsigneds 
     << "  index of existing external halo master node "
     << recv_unsigneds[counter_for_recv_unsigneds]
     << std::endl;
#endif
    // Copy node from received location
    new_master_nod_pt=mesh_pt->external_halo_node_pt
     (loc_p,recv_unsigneds[counter_for_recv_unsigneds++]);
     }
   }
 }

//======start of construct_new_external_halo_master_node_helper===========
/// Helper function which constructs a new external halo master node
/// with the required information sent from the haloed process
//========================================================================
template<class EXT_ELEMENT>
 void Missing_masters_functions::construct_new_external_halo_master_node_helper
 (Node* &new_master_nod_pt, Node* &nod_pt, unsigned& loc_p,
  Mesh* const &mesh_pt,
  unsigned &counter_for_recv_unsigneds,
  Vector<unsigned>& recv_unsigneds,
  unsigned &counter_for_recv_doubles,
  Vector<double>& recv_doubles)
 {
  // First three sent numbers are dimension, position type and nvalue
  // (to be used in Node constructors)
#ifdef ANNOTATE_MISSING_MASTERS_COMMUNICATION
  oomph_info 
   << "Rec:" << counter_for_recv_unsigneds 
   << "  ndim for external halo master node "
   << recv_unsigneds[counter_for_recv_unsigneds]
   << std::endl;
#endif
  unsigned n_dim=recv_unsigneds[counter_for_recv_unsigneds++];
#ifdef ANNOTATE_MISSING_MASTERS_COMMUNICATION
  oomph_info 
   << "Rec:" << counter_for_recv_unsigneds 
   << "  nposition type for external halo master node "
   << recv_unsigneds[counter_for_recv_unsigneds]
   << std::endl;
#endif
  unsigned n_position_type=recv_unsigneds
   [counter_for_recv_unsigneds++];
#ifdef ANNOTATE_MISSING_MASTERS_COMMUNICATION
  oomph_info 
   << "Rec:" << counter_for_recv_unsigneds 
   << "  nvalue for external halo master node "
   << recv_unsigneds[counter_for_recv_unsigneds]
   << std::endl;
#endif
  unsigned n_value=recv_unsigneds
   [counter_for_recv_unsigneds++];
#ifdef ANNOTATE_MISSING_MASTERS_COMMUNICATION
  oomph_info 
   << "Rec:" << counter_for_recv_unsigneds 
   << "  non-halo processor ID for external halo master node "
   << recv_unsigneds[counter_for_recv_unsigneds]
   << std::endl;
#endif
  unsigned non_halo_proc_ID=recv_unsigneds
   [counter_for_recv_unsigneds++];

  // If it's a solid node also receive the lagrangian dimension and pos type
  SolidNode* solid_nod_pt=dynamic_cast<SolidNode*>(nod_pt);
  unsigned n_lag_dim;
  unsigned n_lag_type;
  if (solid_nod_pt!=0)
   {
#ifdef ANNOTATE_MISSING_MASTERS_COMMUNICATION
    oomph_info 
     << "Rec:" << counter_for_recv_unsigneds 
     << "  nlagrdim for external halo master solid node "
     << recv_unsigneds[counter_for_recv_unsigneds]
     << std::endl;
#endif
    n_lag_dim=recv_unsigneds[counter_for_recv_unsigneds++];
#ifdef ANNOTATE_MISSING_MASTERS_COMMUNICATION
    oomph_info 
     << "Rec:" << counter_for_recv_unsigneds 
     << "  nlagrtype for external halo master solid node "
     << recv_unsigneds[counter_for_recv_unsigneds]
     << std::endl;
#endif
    n_lag_type=recv_unsigneds[counter_for_recv_unsigneds++];
   }

  // Null TimeStepper for now
  TimeStepper* time_stepper_pt=0;
  // Default number of previous values to 1
  unsigned n_prev=1;

  //Just take timestepper from a node
  //Let's use first node of first element since this must exist
  time_stepper_pt=mesh_pt->
   finite_element_pt(0)->node_pt(0)->time_stepper_pt();

  // Is the node for which the master is required Algebraic, Macro or Solid?
  AlgebraicNode* alg_nod_pt=dynamic_cast<AlgebraicNode*>(nod_pt);
  MacroElementNodeUpdateNode* macro_nod_pt=
   dynamic_cast<MacroElementNodeUpdateNode*>(nod_pt);

  // What type of node was the node for which we are constructing a master?
  if (alg_nod_pt!=0)
   {
    // The master node should also be algebraic
#ifdef ANNOTATE_MISSING_MASTERS_COMMUNICATION
    oomph_info 
     << "Rec:" << counter_for_recv_unsigneds 
     << "  Boolean for algebraic boundary node "
     << recv_unsigneds[counter_for_recv_unsigneds]
     << std::endl;
#endif
    // If this master node's haloed copy is on a boundary then 
    // it needs to be on the same boundary here
    if (recv_unsigneds[counter_for_recv_unsigneds++]==1)
     {
      // Create a new BoundaryNode (not attached to an element)
      if (time_stepper_pt!=0)
       {
        new_master_nod_pt = new BoundaryNode<AlgebraicNode>
         (time_stepper_pt,n_dim,n_position_type,n_value);
       }
      else
       {
        new_master_nod_pt = new BoundaryNode<AlgebraicNode>
         (n_dim,n_position_type,n_value);
       }

      // How many boundaries does the algebraic master node live on?
#ifdef ANNOTATE_MISSING_MASTERS_COMMUNICATION
      oomph_info << "Rec:" << counter_for_recv_unsigneds 
                 << " Number of boundaries the algebraic master node is on: " 
                 << recv_unsigneds[counter_for_recv_unsigneds]
                 << std::endl;
#endif
      unsigned nb=recv_unsigneds[counter_for_recv_unsigneds++];
      for (unsigned i=0;i<nb;i++)
       {
        // Boundary number
#ifdef ANNOTATE_MISSING_MASTERS_COMMUNICATION
        oomph_info << "Rec:" << counter_for_recv_unsigneds 
                   << "  Algebraic master node is on boundary " 
                   << recv_unsigneds[counter_for_recv_unsigneds]
                   << std::endl;
#endif
        unsigned i_bnd=
         recv_unsigneds[counter_for_recv_unsigneds++];
        mesh_pt->add_boundary_node(i_bnd,new_master_nod_pt);
       }

      
      // Do we have additional values created by face elements?
#ifdef ANNOTATE_MISSING_MASTERS_COMMUNICATION
      oomph_info 
       << "Rec:" << counter_for_recv_unsigneds << " "
       << "Number of additional values created by face element "
       << "for master node " 
       << recv_unsigneds[counter_for_recv_unsigneds]
       << std::endl;
#endif
      unsigned n_entry=recv_unsigneds[
       counter_for_recv_unsigneds++];
      if (n_entry>0)
       {
        // Create storage, if it doesn't already exist, for the map 
        // that will contain the position of the first entry of 
        // this face element's additional values, 
        BoundaryNodeBase* bnew_master_nod_pt=
         dynamic_cast<BoundaryNodeBase*>(new_master_nod_pt);
#ifdef PARANOID
        if (bnew_master_nod_pt==0)
         {
          throw OomphLibError(
           "Failed to cast new node to boundary node\n",
           OOMPH_CURRENT_FUNCTION,
           OOMPH_EXCEPTION_LOCATION);
         }
#endif
        if(bnew_master_nod_pt->
           index_of_first_value_assigned_by_face_element_pt()==0)
         {
          bnew_master_nod_pt->
           index_of_first_value_assigned_by_face_element_pt()= 
           new std::map<unsigned, unsigned>; 
         }
        
        // Get pointer to the map of indices associated with
        // additional values created by face elements
        std::map<unsigned, unsigned>* map_pt=
         bnew_master_nod_pt->index_of_first_value_assigned_by_face_element_pt();
        
        // Loop over number of entries in map
        for (unsigned i=0;i<n_entry;i++)
         {
          // Read out pairs...
          
#ifdef ANNOTATE_MISSING_MASTERS_COMMUNICATION
          oomph_info << "Rec:" << counter_for_recv_unsigneds 
                     << " Key of map entry for master node" 
                     << recv_unsigneds[counter_for_recv_unsigneds]
                     << std::endl;
#endif
          unsigned first=recv_unsigneds[
           counter_for_recv_unsigneds++];
          
#ifdef ANNOTATE_MISSING_MASTERS_COMMUNICATION
          oomph_info << "Rec:" << counter_for_recv_unsigneds 
                     << " Value of map entry for master node" 
                     << recv_unsigneds[counter_for_recv_unsigneds]
                     << std::endl;
#endif
          unsigned second=recv_unsigneds[
           counter_for_recv_unsigneds++];
          
          // ...and assign
          (*map_pt)[first]=second;
         }
       }

     }
    else
     {
      // Create node (not attached to any element)
      if (time_stepper_pt!=0)
       {
        new_master_nod_pt = new AlgebraicNode
         (time_stepper_pt,n_dim,n_position_type,n_value);
       }
      else
       {
        new_master_nod_pt = new AlgebraicNode
         (n_dim,n_position_type,n_value);
       }
     }

    // Add this as an external halo node BEFORE considering node update!
    mesh_pt->add_external_halo_node_pt(loc_p,new_master_nod_pt);

    // The external mesh is itself Algebraic...
    AlgebraicMesh* alg_mesh_pt=dynamic_cast<AlgebraicMesh*>
     (mesh_pt);

#ifdef ANNOTATE_MISSING_MASTERS_COMMUNICATION    
    oomph_info 
     << "Rec:" << counter_for_recv_unsigneds 
     << "  algebraic node update id for master node " 
     << recv_unsigneds[counter_for_recv_unsigneds]
     << std::endl;
#endif
    /// The first entry of All_unsigned_values is the default node update id
    unsigned update_id=recv_unsigneds
     [counter_for_recv_unsigneds++];

    // Setup algebraic node update info for this new node
    Vector<double> ref_value;

#ifdef ANNOTATE_MISSING_MASTERS_COMMUNICATION
    oomph_info 
     << "Rec:" << counter_for_recv_unsigneds 
     << "  algebraic node number of ref values for master node " 
     << recv_unsigneds[counter_for_recv_unsigneds]
     << std::endl;
#endif
    // The size of this vector is in the next entry
    unsigned n_ref_val=recv_unsigneds
     [counter_for_recv_unsigneds++];

    // The reference values are in the subsequent entries of All_double_values
    ref_value.resize(n_ref_val);
    for (unsigned i_ref=0;i_ref<n_ref_val;i_ref++)
     {
      ref_value[i_ref]=recv_doubles
       [counter_for_recv_doubles++];
     }

    // Also require a Vector of geometric objects
    Vector<GeomObject*> geom_object_pt;

#ifdef ANNOTATE_MISSING_MASTERS_COMMUNICATION
    oomph_info 
     << "Rec:" << counter_for_recv_unsigneds 
     << "  algebraic node number of geom objects for master node " 
     << recv_unsigneds[counter_for_recv_unsigneds]
     << std::endl;
#endif

    // The size of this vector is in the next entry of All_unsigned_values
    unsigned n_geom_obj=recv_unsigneds
     [counter_for_recv_unsigneds++];

    // The remaining indices are in the rest of 
    // All_alg_nodal_info
    geom_object_pt.resize(n_geom_obj);
    for (unsigned i_geom=0;i_geom<n_geom_obj;i_geom++)
     {
#ifdef ANNOTATE_MISSING_MASTERS_COMMUNICATION
      oomph_info 
       << "Rec:" << counter_for_recv_unsigneds 
       << "  algebraic node: " << i_geom << "-th out of "
       << n_geom_obj << "-th geom index " 
       << recv_unsigneds[counter_for_recv_unsigneds]
       << std::endl;
#endif
      unsigned geom_index=recv_unsigneds
       [counter_for_recv_unsigneds++];

      // This index indicates which (if any) of the AlgebraicMesh's
      // stored geometric objects should be used
      geom_object_pt[i_geom]=alg_mesh_pt->geom_object_list_pt(geom_index);
     }

    AlgebraicNode* alg_master_nod_pt=
     dynamic_cast<AlgebraicNode*>(new_master_nod_pt);

    /// ... so for the specified update_id, call
    /// add_node_update_info
    alg_master_nod_pt->add_node_update_info
     (update_id,alg_mesh_pt,geom_object_pt,ref_value);

    /// Now call update_node_update
    alg_mesh_pt->update_node_update(alg_master_nod_pt);
   }
  else if (macro_nod_pt!=0)
   {
    // The master node should also be a macro node
    // If this master node's haloed copy is on a boundary then 
    // it needs to be on the same boundary here
#ifdef ANNOTATE_MISSING_MASTERS_COMMUNICATION
    oomph_info << "Rec:" << counter_for_recv_unsigneds 
               << "  Boolean for master algebraic node is boundary node "
               << recv_unsigneds[counter_for_recv_unsigneds]
               << std::endl;
#endif
    if (recv_unsigneds[counter_for_recv_unsigneds++]==1)
     {
      // Create a new BoundaryNode (not attached to an element)
      if (time_stepper_pt!=0)
       {
        new_master_nod_pt = new BoundaryNode<MacroElementNodeUpdateNode>
         (time_stepper_pt,n_dim,n_position_type,n_value);
       }
      else
       {
        new_master_nod_pt = new BoundaryNode<MacroElementNodeUpdateNode>
         (n_dim,n_position_type,n_value);
       }


      // How many boundaries does the macro element master node live on?
#ifdef ANNOTATE_MISSING_MASTERS_COMMUNICATION
      oomph_info 
       << "Rec:" << counter_for_recv_unsigneds 
       << " Number of boundaries the macro element master node is on: " 
       << recv_unsigneds[counter_for_recv_unsigneds]
       << std::endl;
#endif
      unsigned nb=recv_unsigneds[counter_for_recv_unsigneds++];
      for (unsigned i=0;i<nb;i++)
       {
        // Boundary number
#ifdef ANNOTATE_MISSING_MASTERS_COMMUNICATION
        oomph_info << "Rec:" << counter_for_recv_unsigneds 
                   << "  Macro element master node is on boundary " 
                   << recv_unsigneds[counter_for_recv_unsigneds]
                   << std::endl;
#endif
        unsigned i_bnd=
         recv_unsigneds[counter_for_recv_unsigneds++];
        mesh_pt->add_boundary_node(i_bnd,new_master_nod_pt);
       }
      
      // Do we have additional values created by face elements?
#ifdef ANNOTATE_MISSING_MASTERS_COMMUNICATION
      oomph_info 
       << "Rec:" << counter_for_recv_unsigneds 
       << " Number of additional values created by face element "
       << "for macro element master node " 
       << recv_unsigneds[counter_for_recv_unsigneds]
       << std::endl;
#endif
      unsigned n_entry=recv_unsigneds[
       counter_for_recv_unsigneds++];
      if (n_entry>0)
       {
        // Create storage, if it doesn't already exist, for the map 
        // that will contain the position of the first entry of 
        // this face element's additional values, 
        BoundaryNodeBase* bnew_master_nod_pt=
         dynamic_cast<BoundaryNodeBase*>(new_master_nod_pt);
#ifdef PARANOID
        if (bnew_master_nod_pt==0)
         {
          throw OomphLibError(
           "Failed to cast new node to boundary node\n",
           OOMPH_CURRENT_FUNCTION,
           OOMPH_EXCEPTION_LOCATION);
         }
#endif
        if(bnew_master_nod_pt->
           index_of_first_value_assigned_by_face_element_pt()==0)
         {
          bnew_master_nod_pt->
           index_of_first_value_assigned_by_face_element_pt()= 
           new std::map<unsigned, unsigned>; 
         }
        
        // Get pointer to the map of indices associated with
        // additional values created by face elements
        std::map<unsigned, unsigned>* map_pt=
         bnew_master_nod_pt->index_of_first_value_assigned_by_face_element_pt();
        
        // Loop over number of entries in map
        for (unsigned i=0;i<n_entry;i++)
         {
          // Read out pairs...
          
#ifdef ANNOTATE_MISSING_MASTERS_COMMUNICATION
          oomph_info << "Rec:" << counter_for_recv_unsigneds 
                     << " Key of map entry for macro element master node" 
                     << recv_unsigneds[counter_for_recv_unsigneds]
                     << std::endl;
#endif
          unsigned first=recv_unsigneds[
           counter_for_recv_unsigneds++];
          
#ifdef ANNOTATE_MISSING_MASTERS_COMMUNICATION
          oomph_info << "Rec:" << counter_for_recv_unsigneds 
                     << " Value of map entry for macro element master node" 
                     << recv_unsigneds[counter_for_recv_unsigneds]
                     << std::endl;
#endif
          unsigned second=recv_unsigneds[
           counter_for_recv_unsigneds++];
          
          // ...and assign
          (*map_pt)[first]=second;
         }
       }

     }
    else
     {
      // Create node (not attached to any element)
      if (time_stepper_pt!=0)
       {
        new_master_nod_pt = new MacroElementNodeUpdateNode
         (time_stepper_pt,n_dim,n_position_type,n_value);
       }
      else
       {
        new_master_nod_pt = new MacroElementNodeUpdateNode
         (n_dim,n_position_type,n_value);
       }
     }

    // Add this as an external halo node
    mesh_pt->add_external_halo_node_pt(loc_p,new_master_nod_pt);
    oomph_info << "Added external halo master node:" << new_master_nod_pt << " at [ " << new_master_nod_pt->x(0) << ", " << new_master_nod_pt->x(1) << " ]" << std::endl;

    // Create a new node update element for this master node if required
    FiniteElement *new_node_update_f_el_pt=0;
#ifdef ANNOTATE_MISSING_MASTERS_COMMUNICATION
    oomph_info << "Rec:" << counter_for_recv_unsigneds 
               << "  Bool: need new external halo element "
               << recv_unsigneds[counter_for_recv_unsigneds]
               << std::endl;
#endif
    if (recv_unsigneds[counter_for_recv_unsigneds++]==1)
     {
      // Issue warning about adding a macro element to the external storage
      std::ostringstream warn_stream;
      warn_stream << "You are adding a MacroElementNodeUpdate element to the\n"
                  << "external storage. This functionality is still being\n"
                  << "developed and may cause problems later on, say during\n"
                  << "Problem::remove_duplicate_data().";
      OomphLibWarning(
       warn_stream.str(),
       "Missing_masters_functions::construct_new_external_halo_master_node_helper()",
       OOMPH_EXCEPTION_LOCATION);

      GeneralisedElement* new_node_update_el_pt = new EXT_ELEMENT;

      //Add external halo element to this mesh
      mesh_pt->add_external_halo_element_pt(
       loc_p,new_node_update_el_pt);

      //Cast to finite element
      new_node_update_f_el_pt = 
       dynamic_cast<FiniteElement*>(new_node_update_el_pt);

      // Need number of interpolated values if Refineable
      int n_cont_inter_values;
      if (dynamic_cast<RefineableElement*>(new_node_update_f_el_pt)!=0)
       {
        n_cont_inter_values=dynamic_cast<RefineableElement*>
         (new_node_update_f_el_pt)->ncont_interpolated_values();
       }
      else
       {
        n_cont_inter_values=-1;
       }
#ifdef ANNOTATE_MISSING_MASTERS_COMMUNICATION
      oomph_info << "Rec:" << counter_for_recv_unsigneds 
                 << "  Bool: we have a macro element mesh "
                 << recv_unsigneds[counter_for_recv_unsigneds]
                 << std::endl;
#endif
      // If we're using macro elements to update,
      if (recv_unsigneds[counter_for_recv_unsigneds++]==1)
       {
        // Set the macro element
        MacroElementNodeUpdateMesh* macro_mesh_pt=
         dynamic_cast<MacroElementNodeUpdateMesh*>
         (mesh_pt);

#ifdef ANNOTATE_MISSING_MASTERS_COMMUNICATION
      oomph_info << "Rec:" << counter_for_recv_unsigneds 
                 << "  Number of macro element "
                 << recv_unsigneds[counter_for_recv_unsigneds]
                 << std::endl;
#endif
        unsigned macro_el_num=
         recv_unsigneds[counter_for_recv_unsigneds++];
        new_node_update_f_el_pt->set_macro_elem_pt
         (macro_mesh_pt->macro_domain_pt()->macro_element_pt(macro_el_num));

        // we need to receive
        // the lower left and upper right coordinates of the macro
        QElementBase* q_el_pt=
         dynamic_cast<QElementBase*>(new_node_update_f_el_pt);
        if (q_el_pt!=0)
         {
          unsigned el_dim=q_el_pt->dim();
          for (unsigned i_dim=0;i_dim<el_dim;i_dim++)
           {
            q_el_pt->s_macro_ll(i_dim)=recv_doubles
             [counter_for_recv_doubles++];
            q_el_pt->s_macro_ur(i_dim)=recv_doubles
             [counter_for_recv_doubles++];
           }
         }
        else // Throw an error
         {
          std::ostringstream error_stream;
          error_stream << "You are using a MacroElement node update\n"
                       << "in a case with non-QElements. This has not\n"
                       << "yet been implemented.\n";
          throw OomphLibError
           (error_stream.str(),
            OOMPH_CURRENT_FUNCTION,
            OOMPH_EXCEPTION_LOCATION);
         }
       }

      // Check if haloed version was p-refineable
#ifdef ANNOTATE_MISSING_MASTERS_COMMUNICATION
      oomph_info << "Rec:" << counter_for_recv_unsigneds 
                 << "  Element was p-refineable "
                 << recv_unsigneds[counter_for_recv_unsigneds]
                 << std::endl;
#endif
      unsigned el_was_p_refineable=
       recv_unsigneds[counter_for_recv_unsigneds++];
      if(el_was_p_refineable)
       {
        // Check created element is p-refineable
        PRefineableElement* p_refineable_el_pt =
         dynamic_cast<PRefineableElement*>(new_node_update_f_el_pt);
        if(p_refineable_el_pt!=0)
         {
          // Recieve p-order
#ifdef ANNOTATE_MISSING_MASTERS_COMMUNICATION
          oomph_info << "Rec:" << counter_for_recv_unsigneds 
                     << "  p-order: "
                     << recv_unsigneds[counter_for_recv_unsigneds]
                     << std::endl;
#endif
          unsigned p_order=
           recv_unsigneds[counter_for_recv_unsigneds++];

          // Do initial setup with original element as the clone's adopted father
          p_refineable_el_pt->initial_setup(0,p_order);
          //BENFLAG:
          oomph_info << "New node update element: " << new_node_update_el_pt << " (p-order = " << p_order << ")" << std::endl;
         }
        else
         {
          std::ostringstream error_stream;
          error_stream << "Created MacroElement node update element is not p-refineable\n"
                       << "but the haloed version is.\n";
          throw OomphLibError
           (error_stream.str(),
            "Missing_masters_functions::construct_new_external_halo_master_...()",
            OOMPH_EXCEPTION_LOCATION);
         }
       }

      unsigned n_node=new_node_update_f_el_pt->nnode();
      for (unsigned j=0;j<n_node;j++)
       {
        Node* new_nod_pt=0;
        add_external_halo_node_to_storage<EXT_ELEMENT>
         (new_nod_pt,mesh_pt,loc_p,j,new_node_update_f_el_pt,
          n_cont_inter_values,
          counter_for_recv_unsigneds,
          recv_unsigneds,
          counter_for_recv_doubles,
          recv_doubles);
        // BENFLAG:
        oomph_info << "Added node " << new_nod_pt << " at [ " << new_nod_pt->x(0) << ", " << new_nod_pt->x(1) << " ]" << std::endl;
       }

      //BENFLAG:
      oomph_info << "New node update element: " << new_node_update_f_el_pt << " (nnode_1d = " << new_node_update_f_el_pt->nnode_1d() << ")" << std::endl;
     }
    else // The node update element exists already
     {
#ifdef ANNOTATE_MISSING_MASTERS_COMMUNICATION
      oomph_info << "Rec:" << counter_for_recv_unsigneds 
                 << "  Found internally? "
                 << recv_unsigneds[counter_for_recv_unsigneds]
                 << std::endl;
#endif
      unsigned found_internally = recv_unsigneds[counter_for_recv_unsigneds++];
#ifdef ANNOTATE_MISSING_MASTERS_COMMUNICATION
      oomph_info << "Rec:" << counter_for_recv_unsigneds 
                 << "  Number of already existing external halo element "
                 << recv_unsigneds[counter_for_recv_unsigneds]
                 << std::endl;
#endif
      unsigned halo_element_index = recv_unsigneds[counter_for_recv_unsigneds++];
      if(found_internally!=0)
       {
        new_node_update_f_el_pt=dynamic_cast<FiniteElement*>(
         (mesh_pt->halo_element_pt(loc_p))[halo_element_index]);
        //BENFLAG:
        oomph_info << "Existing node update element: " << new_node_update_f_el_pt << " (nnode_1d = " << new_node_update_f_el_pt->nnode_1d() << ")" << std::endl;
        oomph_info << "on proc " << loc_p << " at (internal) index " << halo_element_index << std::endl;

//        //BENFLAG: Also add halo element to external storage
//        oomph_info << "Adding to external halo storage..." << std::endl;
//        GeneralisedElement* g_el_pt = dynamic_cast<GeneralisedElement*>(new_node_update_f_el_pt);
//        mesh_pt->add_external_halo_element_pt(
//         loc_p,g_el_pt);
//
//        // Check if also found externally
//#ifdef ANNOTATE_MISSING_MASTERS_COMMUNICATION
//        oomph_info << "Rec:" << counter_for_recv_unsigneds 
//                   << "  Found externally too? "
//                   << recv_unsigneds[counter_for_recv_unsigneds]
//                   << std::endl;
//#endif
//        unsigned found_externally_too = recv_unsigneds[counter_for_recv_unsigneds++];
//        std::cout << "received found_externally_too = " << found_externally_too << std::endl;
//        if(found_externally_too==1234)
//         {
//#ifdef ANNOTATE_MISSING_MASTERS_COMMUNICATION
//          oomph_info << "Rec:" << counter_for_recv_unsigneds 
//                     << "  Number of already existing external halo element "
//                     << recv_unsigneds[counter_for_recv_unsigneds]
//                     << std::endl;
//#endif
//          unsigned ext_version_halo_element_index = recv_unsigneds[counter_for_recv_unsigneds++];
//          std::cout << "received ext_version_halo_element_index = " << ext_version_halo_element_index << std::endl;
//
//          FiniteElement* ext_version_pt = dynamic_cast<FiniteElement*>(
//           (mesh_pt->halo_element_pt(loc_p))[ext_version_halo_element_index]);
//          //BENFLAG:
//          oomph_info << "Existing node update element: " << ext_version_pt << " (nnode_1d = " << ext_version_pt->nnode_1d() << ")" << std::endl;
//          oomph_info << "on proc " << loc_p << " is also at (external) index " << ext_version_halo_element_index << std::endl;
//          for(unsigned j=0; j<ext_version_pt->nnode(); j++)
//           {
//            oomph_info << ext_version_pt->node_pt(j) << " at [ " << ext_version_pt->node_pt(j)->x(0) << ", " << ext_version_pt->node_pt(j)->x(1) << " ]" << std::endl;
//           }
//         }
        
       }
      else
       {
        new_node_update_f_el_pt=dynamic_cast<FiniteElement*>(
         mesh_pt->external_halo_element_pt
         (loc_p,halo_element_index));
        //BENFLAG:
        oomph_info << "Existing node update element: " << new_node_update_f_el_pt << " (nnode_1d = " << new_node_update_f_el_pt->nnode_1d() << ")" << std::endl;
        oomph_info << "on proc " << loc_p << " at (external) index " << recv_unsigneds[counter_for_recv_unsigneds-1] << std::endl;
        //oomph_info << "...and doesn't exist in the external storage." << std::endl;
       }
     }

    // Remaining required information to create functioning
    // MacroElementNodeUpdateNode...

    // Get the required geom objects for the node update
    // from the mesh
    Vector<GeomObject*> geom_object_vector_pt;
    MacroElementNodeUpdateMesh* macro_mesh_pt=
     dynamic_cast<MacroElementNodeUpdateMesh*>
     (mesh_pt);
    geom_object_vector_pt=macro_mesh_pt->geom_object_vector_pt();

    // Cast to MacroElementNodeUpdateNode
    MacroElementNodeUpdateNode* macro_master_nod_pt=
     dynamic_cast<MacroElementNodeUpdateNode*>(new_master_nod_pt);

    // Set all required information - node update element,
    // local coordinate in this element, and then set node update info
    macro_master_nod_pt->node_update_element_pt()=
     new_node_update_f_el_pt;

    

    ////print out nodes
    //std::cout << "nodes are:" << std::endl;
    //for(unsigned j=0; j<new_node_update_f_el_pt->nnode(); j++)
    // {
    //  std::cout << new_node_update_f_el_pt->node_pt(j) << " at [ " << new_node_update_f_el_pt->node_pt(j)->x(0) << ", " << new_node_update_f_el_pt->node_pt(j)->x(1) << " ]" << std::endl;
    //  //std::cout << new_node_update_f_el_pt->node_pt(j) << std::endl;
    // }
    //std::cout << "should include: " << macro_master_nod_pt << " at [ " << macro_master_nod_pt->x(0) << ", " << macro_master_nod_pt->x(1) << " ]" << std::endl;

    

    // Need to get the local node index of the macro_master_nod_pt
    unsigned local_node_index=0;
    //std::cout << "before: " << local_node_index << std::endl;
    unsigned n_node=new_node_update_f_el_pt->nnode();
    for (unsigned j=0;j<n_node;j++)
     {
      if (macro_master_nod_pt==new_node_update_f_el_pt->node_pt(j))
       {
        //std::cout << "Node " << macro_master_nod_pt << " found at index " << j << " in update element." << std::endl;
        local_node_index=j;
        break;
       }
      //BENFLAG:
      if(j==n_node-1)
       {
        //// Check if sons...
        //RefineableElement* ref_el_pt = dynamic_cast<RefineableElement*>(new_node_update_f_el_pt);
        //if(ref_el_pt->tree_pt()->nsons()!=0)
        // {
        //  std::cout << "update el has sons!" << std::endl;
        // }
        //else
        // {
        //  std::cout << "No sons." << std::endl;
        // }

        //oomph_info << "Node not found in update element!" << std::endl;
        throw OomphLibError(
               "Node not found in update element!",
               "Missing_masters_functions::construct_new_external_halo_master_node_helper()",
               OOMPH_EXCEPTION_LOCATION);
       }
     }
    //std::cout << "after: " << local_node_index << std::endl;

    Vector<double> s_in_macro_node_update_element;
    new_node_update_f_el_pt->local_coordinate_of_node
     (local_node_index,s_in_macro_node_update_element);

    macro_master_nod_pt->set_node_update_info
     (new_node_update_f_el_pt,s_in_macro_node_update_element,
      geom_object_vector_pt);
   }
  else if (solid_nod_pt!=0)
   {
    // The master node should also be a SolidNode
    // If this node was on a boundary then it needs to
    // be on the same boundary here
#ifdef ANNOTATE_MISSING_MASTERS_COMMUNICATION
      oomph_info << "Rec:" << counter_for_recv_unsigneds 
                 << "  Bool master is a boundary (solid) node "
                 << recv_unsigneds[counter_for_recv_unsigneds]
                 << std::endl;
#endif
    if (recv_unsigneds[counter_for_recv_unsigneds++]==1)
     {
      // Construct a new boundary node
      if (time_stepper_pt!=0)
       {
        new_master_nod_pt=new BoundaryNode<SolidNode>
         (time_stepper_pt,n_lag_dim,n_lag_type,n_dim,n_position_type,n_value);
       }
      else
       {
        new_master_nod_pt=new BoundaryNode<SolidNode>
         (n_lag_dim,n_lag_type,n_dim,n_position_type,n_value);
       }


      // How many boundaries does the macro element master node live on?
#ifdef ANNOTATE_MISSING_MASTERS_COMMUNICATION
      oomph_info 
       << "Rec:" << counter_for_recv_unsigneds 
       << " Number of boundaries the solid master node is on: " 
       << recv_unsigneds[counter_for_recv_unsigneds]
       << std::endl;
#endif
      unsigned nb=recv_unsigneds[counter_for_recv_unsigneds++];
      for (unsigned i=0;i<nb;i++)
       {
        // Boundary number
#ifdef ANNOTATE_MISSING_MASTERS_COMMUNICATION
        oomph_info << "Rec:" << counter_for_recv_unsigneds 
                   << " Solid master node is on boundary " 
                   << recv_unsigneds[counter_for_recv_unsigneds]
                   << std::endl;
#endif
        unsigned i_bnd=
         recv_unsigneds[counter_for_recv_unsigneds++];
        mesh_pt->add_boundary_node(i_bnd,new_master_nod_pt);
       }
      
      // Do we have additional values created by face elements?
#ifdef ANNOTATE_MISSING_MASTERS_COMMUNICATION
      oomph_info 
       << "Rec:" << counter_for_recv_unsigneds 
       << " Number of additional values created by face element "
       << "for solid master node " 
       << recv_unsigneds[counter_for_recv_unsigneds]
       << std::endl;
#endif
      unsigned n_entry=recv_unsigneds[
       counter_for_recv_unsigneds++];
      if (n_entry>0)
       {
        // Create storage, if it doesn't already exist, for the map 
        // that will contain the position of the first entry of 
        // this face element's additional values, 
        BoundaryNodeBase* bnew_master_nod_pt=
         dynamic_cast<BoundaryNodeBase*>(new_master_nod_pt);
#ifdef PARANOID
      if (bnew_master_nod_pt==0)
       {
        throw OomphLibError(
         "Failed to cast new node to boundary node\n",
         OOMPH_CURRENT_FUNCTION,
         OOMPH_EXCEPTION_LOCATION);
       }
#endif
      if(bnew_master_nod_pt->
         index_of_first_value_assigned_by_face_element_pt()==0)
       {
        bnew_master_nod_pt->
         index_of_first_value_assigned_by_face_element_pt()= 
         new std::map<unsigned, unsigned>; 
       }
      
      // Get pointer to the map of indices associated with
      // additional values created by face elements
      std::map<unsigned, unsigned>* map_pt=
       bnew_master_nod_pt->index_of_first_value_assigned_by_face_element_pt();
      
      // Loop over number of entries in map
      for (unsigned i=0;i<n_entry;i++)
       {
        // Read out pairs...
        
#ifdef ANNOTATE_MISSING_MASTERS_COMMUNICATION
        oomph_info << "Rec:" << counter_for_recv_unsigneds 
                   << " Key of map entry for solid master node" 
                   << recv_unsigneds[counter_for_recv_unsigneds]
                   << std::endl;
#endif
        unsigned first=recv_unsigneds[
         counter_for_recv_unsigneds++];
        
#ifdef ANNOTATE_MISSING_MASTERS_COMMUNICATION
        oomph_info << "Rec:" << counter_for_recv_unsigneds 
                   << " Value of map entry for solid master node" 
                   << recv_unsigneds[counter_for_recv_unsigneds]
                   << std::endl;
#endif
        unsigned second=recv_unsigneds[
         counter_for_recv_unsigneds++];
        
        // ...and assign
        (*map_pt)[first]=second;
       }
     }
    
   }
   else
     {
      // Construct an ordinary (non-boundary) node
      if (time_stepper_pt!=0)
       {
        new_master_nod_pt=new SolidNode
         (time_stepper_pt,n_lag_dim,n_lag_type,n_dim,n_position_type,n_value);
       }
      else
       {
        new_master_nod_pt=new SolidNode
         (n_lag_dim,n_lag_type,n_dim,n_position_type,n_value);
       }
     }

    // Add this as an external halo node
    mesh_pt->add_external_halo_node_pt(loc_p,new_master_nod_pt);

    // Copy across particular info required for SolidNode
    // NOTE: Are there any problems with additional values for SolidNodes?
    SolidNode* solid_master_nod_pt=dynamic_cast<SolidNode*>(new_master_nod_pt);
    unsigned n_solid_val=solid_master_nod_pt->variable_position_pt()->nvalue();
    for (unsigned i_val=0;i_val<n_solid_val;i_val++)
     {
      for (unsigned t=0;t<n_prev;t++)
       {
        solid_master_nod_pt->variable_position_pt()->
         set_value(t,i_val,
                   recv_doubles[counter_for_recv_doubles++]);
       }
     }
   }
  else // Just an ordinary node!
   {
#ifdef ANNOTATE_MISSING_MASTERS_COMMUNICATION
    oomph_info << "Rec:" << counter_for_recv_unsigneds 
               << "  Bool node is on boundary "
               << recv_unsigneds[counter_for_recv_unsigneds]
               << std::endl;
#endif

    // If this node was on a boundary then it needs to
    // be on the same boundary here
    if (recv_unsigneds[counter_for_recv_unsigneds++]==1)
     {
      // Construct a new boundary node
      if (time_stepper_pt!=0)
       {
        new_master_nod_pt=new BoundaryNode<Node>
         (time_stepper_pt,n_dim,n_position_type,n_value);
       }
      else
       {
        new_master_nod_pt=new BoundaryNode<Node>
         (n_dim,n_position_type,n_value);
       }
      
      // How many boundaries does the master node live on?
#ifdef ANNOTATE_MISSING_MASTERS_COMMUNICATION
      oomph_info << "Rec:" << counter_for_recv_unsigneds 
                 << " Number of boundaries the master node is on: " 
                 << recv_unsigneds[counter_for_recv_unsigneds]
                 << std::endl;
#endif
      unsigned nb=recv_unsigneds[counter_for_recv_unsigneds++];
      for (unsigned i=0;i<nb;i++)
       {
        // Boundary number
#ifdef ANNOTATE_MISSING_MASTERS_COMMUNICATION
        oomph_info << "Rec:" << counter_for_recv_unsigneds 
                   << "  Master node is on boundary " 
                   << recv_unsigneds[counter_for_recv_unsigneds]
                   << std::endl;
#endif
        unsigned i_bnd=
         recv_unsigneds[counter_for_recv_unsigneds++];
        mesh_pt->add_boundary_node(i_bnd,new_master_nod_pt);
       }
      
      
      // Do we have additional values created by face elements?
#ifdef ANNOTATE_MISSING_MASTERS_COMMUNICATION
      oomph_info 
       << "Rec:" << counter_for_recv_unsigneds 
       << " Number of additional values created by face element "
       << "for master node " 
       << recv_unsigneds[counter_for_recv_unsigneds]
       << std::endl;
#endif
      unsigned n_entry=recv_unsigneds[
       counter_for_recv_unsigneds++];
      if (n_entry>0)
       {
        // Create storage, if it doesn't already exist, for the map 
        // that will contain the position of the first entry of 
        // this face element's additional values, 
        BoundaryNodeBase* bnew_master_nod_pt=
         dynamic_cast<BoundaryNodeBase*>(new_master_nod_pt);
#ifdef PARANOID
        if (bnew_master_nod_pt==0)
         {
          throw OomphLibError(
           "Failed to cast new node to boundary node\n",
           OOMPH_CURRENT_FUNCTION,
           OOMPH_EXCEPTION_LOCATION);
         }
#endif
        if(bnew_master_nod_pt->
           index_of_first_value_assigned_by_face_element_pt()==0)
         {
          bnew_master_nod_pt->
           index_of_first_value_assigned_by_face_element_pt()= 
           new std::map<unsigned, unsigned>; 
         }
        
        // Get pointer to the map of indices associated with
        // additional values created by face elements
        std::map<unsigned, unsigned>* map_pt=
         bnew_master_nod_pt->index_of_first_value_assigned_by_face_element_pt();
        
        // Loop over number of entries in map
        for (unsigned i=0;i<n_entry;i++)
         {
          // Read out pairs...
          
#ifdef ANNOTATE_MISSING_MASTERS_COMMUNICATION
          oomph_info << "Rec:" << counter_for_recv_unsigneds 
                     << " Key of map entry for master node" 
                     << recv_unsigneds[counter_for_recv_unsigneds]
                     << std::endl;
#endif
          unsigned first=recv_unsigneds[
           counter_for_recv_unsigneds++];
          
#ifdef ANNOTATE_MISSING_MASTERS_COMMUNICATION
          oomph_info << "Rec:" << counter_for_recv_unsigneds 
                     << " Value of map entry for master node" 
                     << recv_unsigneds[counter_for_recv_unsigneds]
                     << std::endl;
#endif
          unsigned second=recv_unsigneds[
           counter_for_recv_unsigneds++];
          
          // ...and assign
          (*map_pt)[first]=second;
         }
       }
     }
    else
     {
      // Construct an ordinary (non-boundary) node
      if (time_stepper_pt!=0)
       {
        new_master_nod_pt=new Node
         (time_stepper_pt,n_dim,n_position_type,n_value);
       }
      else
       {
        new_master_nod_pt=new Node(n_dim,n_position_type,n_value);
       }
     }

    // Add this as an external halo node
    mesh_pt->add_external_halo_node_pt(loc_p,new_master_nod_pt);
   }

  // Remaining info received for all node types
  // Get copied history values
  //  unsigned n_val=new_master_nod_pt->nvalue();
  for (unsigned i_val=0;i_val<n_value;i_val++)
   {
    for (unsigned t=0;t<n_prev;t++)
     {
      new_master_nod_pt->set_value(t,i_val,recv_doubles
                                   [counter_for_recv_doubles++]);
     }
   }

  // Get copied history values for positions
  unsigned n_nod_dim=new_master_nod_pt->ndim();
  for (unsigned idim=0;idim<n_nod_dim;idim++)
   {
    for (unsigned t=0;t<n_prev;t++)
     {
      // Copy to coordinate
      new_master_nod_pt->x(t,idim)=recv_doubles
       [counter_for_recv_doubles++];
     }
   }

  // Assign correct non-halo processor ID
  new_master_nod_pt->set_halo(non_halo_proc_ID);

 }


#endif


}

#endif