<commit_before>// The libMesh Finite Element Library.
// Copyright (C) 2002-2017 <NAME>, <NAME>, <NAME>

// This library is free software; you can redistribute it and/or
// modify it under the terms of the GNU Lesser General Public
// License as published by the Free Software Foundation; either
// version 2.1 of the License, or (at your option) any later version.

// This library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
// Lesser General Public License for more details.

// You should have received a copy of the GNU Lesser General Public
// License along with this library; if not, write to the Free Software
// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA

#include "libmesh/checkpoint_io.h"

// C++ includes
#include <iostream>
#include <iomanip>
#include <cstdio>
#include <vector>
#include <string>
#include <fstream>
#include <sstream> // for ostringstream

// Local includes
#include "libmesh/boundary_info.h"
#include "libmesh/distributed_mesh.h"
#include "libmesh/elem.h"
#include "libmesh/enum_xdr_mode.h"
#include "libmesh/libmesh_logging.h"
#include "libmesh/mesh_base.h"
#include "libmesh/mesh_communication.h"
#include "libmesh/mesh_tools.h"
#include "libmesh/node.h"
#include "libmesh/parallel.h"
#include "libmesh/partitioner.h"
#include "libmesh/remote_elem.h"
#include "libmesh/xdr_io.h"
#include "libmesh/xdr_cxx.h"

namespace libMesh
{

// ------------------------------------------------------------
// CheckpointIO members
CheckpointIO::CheckpointIO (MeshBase & mesh, const bool binary_in) :
  MeshInput<MeshBase> (mesh,/* is_parallel_format = */ true),
  MeshOutput<MeshBase>(mesh,/* is_parallel_format = */ true),
  ParallelObject      (mesh),
  _binary             (binary_in),
  _parallel           (false),
  _version            ("checkpoint-1.1"),
  _my_processor_id    (processor_id()),
  _my_n_processors    (n_processors())
{
}



CheckpointIO::CheckpointIO (const MeshBase & mesh, const bool binary_in) :
  MeshOutput<MeshBase>(mesh,/* is_parallel_format = */ true),
  ParallelObject      (mesh),
  _binary             (binary_in),
  _parallel           (false),
  _my_processor_id    (processor_id()),
  _my_n_processors    (n_processors())
{
}



CheckpointIO::~CheckpointIO ()
{
}




void CheckpointIO::write (const std::string & name)
{
  LOG_SCOPE("write()", "CheckpointIO");

  // convenient reference to our mesh
  const MeshBase & mesh = MeshOutput<MeshBase>::mesh();

  // Try to dynamic cast the mesh to see if it's a DistributedMesh object
  // Note: Just using is_serial() is not good enough because the Mesh won't
  // have been prepared yet when is when that flag gets set to false... sigh.
  _parallel = _parallel || !mesh.is_replicated();

  // If this is a serial mesh then we're only going to write it on processor 0
  if (_parallel || _my_processor_id == 0)
    {
      std::ostringstream file_name_stream;

      file_name_stream << name;

      if (_parallel)
        file_name_stream << "-" << _my_n_processors << "-" << _my_processor_id;

      Xdr io (file_name_stream.str(), this->binary() ? ENCODE : WRITE);

      // write the version
      io.data(_version, "# version");

      // Write out whether or not this is a serial mesh (helps with error checking on read)
      {
        unsigned int mesh_dimension = mesh.mesh_dimension();
        io.data(mesh_dimension, "# dimensions");
      }

      // Write out whether or not this is a serial mesh (helps with error checking on read)
      {
        unsigned int parallel = _parallel;
        io.data(parallel, "# parallel");
      }

      // If we're writing out a parallel mesh then we need to write the number of processors
      // so we can check it upon reading the file
      if (_parallel)
        {
          largest_id_type n_procs = _my_n_processors;
          io.data(n_procs, "# n_procs");
        }

      // write subdomain names
      this->write_subdomain_names(io);

      // write the nodal locations
      this->write_nodes (io);

      // write connectivity
      this->write_connectivity (io);

      // write the boundary condition information
      this->write_bcs (io);

      // write the nodeset information
      this->write_nodesets (io);

      // write remote_elem connectivity
      this->write_remote_elem (io);

      // close it up
      io.close();
    }

  // this->comm().barrier();
}

void CheckpointIO::write_subdomain_names(Xdr & io) const
{
  {
    const MeshBase & mesh = MeshOutput<MeshBase>::mesh();

    const std::map<subdomain_id_type, std::string> & subdomain_map = mesh.get_subdomain_name_map();

    std::vector<header_id_type> subdomain_ids;   subdomain_ids.reserve(subdomain_map.size());
    std::vector<std::string>  subdomain_names; subdomain_names.reserve(subdomain_map.size());

    // We need to loop over the map and make sure that there aren't any invalid entries.  Since we
    // return writable references in mesh_base, it's possible for the user to leave some entity names
    // blank.  We can't write those to the XDA file.
    header_id_type n_subdomain_names = 0;
    std::map<subdomain_id_type, std::string>::const_iterator it_end = subdomain_map.end();
    for (std::map<subdomain_id_type, std::string>::const_iterator it = subdomain_map.begin(); it != it_end; ++it)
      {
        if (!it->second.empty())
          {
            n_subdomain_names++;
            subdomain_ids.push_back(it->first);
            subdomain_names.push_back(it->second);
          }
      }

    io.data(n_subdomain_names, "# subdomain id to name map");
    // Write out the ids and names in two vectors
    if (n_subdomain_names)
      {
        io.data(subdomain_ids);
        io.data(subdomain_names);
      }
  }
}



void CheckpointIO::write_nodes (Xdr & io) const
{
  // convenient reference to our mesh
  const MeshBase & mesh = MeshOutput<MeshBase>::mesh();

  unsigned int n_nodes_here =
    std::distance(mesh.nodes_begin(), mesh.nodes_end());

  io.data(n_nodes_here, "# n_nodes on proc");

  // Will hold the node id and pid
  std::vector<largest_id_type> id_pid(2);

  // For the coordinates
  std::vector<Real> coords(LIBMESH_DIM);

  MeshBase::const_node_iterator
    it  = mesh.nodes_begin(),
    end = mesh.nodes_end();

  for (; it != end; ++it)
    {
      const Node & node = **it;

      id_pid[0] = node.id();
      id_pid[1] = node.processor_id();

      io.data_stream(&id_pid[0], 2, 2);

#ifdef LIBMESH_ENABLE_UNIQUE_ID
      largest_id_type unique_id = node.unique_id();

      io.data(unique_id, "# unique id");
#endif

      coords[0] = node(0);

#if LIBMESH_DIM > 1
      coords[1] = node(1);
#endif

#if LIBMESH_DIM > 2
      coords[2] = node(2);
#endif

      io.data_stream(&coords[0], LIBMESH_DIM, 3);
    }
}



void CheckpointIO::write_connectivity (Xdr & io) const
{
  libmesh_assert (io.writing());

  // convenient reference to our mesh
  const MeshBase & mesh = MeshOutput<MeshBase>::mesh();

  // We will only write active elements and their parents.
  unsigned int _n_active_levels = n_active_levels_on_processor(mesh);

  std::vector<xdr_id_type> n_elem_at_level(_n_active_levels, 0);

  // Find the number of elements at each level
  {
    MeshBase::const_element_iterator
      it  = mesh.elements_begin(),
      end = mesh.elements_end();

    for (; it != end; ++it)
      n_elem_at_level[(*it)->level()]++;
  }

  io.data(_n_active_levels, "# n_active_levels");

  // Put these out here to reduce memory churn
  // id type pid subdomain_id parent_id
  std::vector<largest_id_type> elem_data(5);
  std::vector<largest_id_type> conn_data;

  for (unsigned int level=0; level < _n_active_levels; level++)
    {
      std::ostringstream comment;
      comment << "# n_elem at level ";
      comment << level ;
      io.data (n_elem_at_level[level], comment.str().c_str());

      MeshBase::const_element_iterator
        it  = mesh.elements_begin(),
        end = mesh.elements_end();

      for (; it != end; ++it)
        {
          const Elem & elem = **it;

          // Only write elements of the current level.
          if (elem.level() != level)
            continue;

          unsigned int n_nodes = elem.n_nodes();

          elem_data[0] = elem.id();
          elem_data[1] = elem.type();
          elem_data[2] = elem.processor_id();
          elem_data[3] = elem.subdomain_id();

          if (elem.parent() != libmesh_nullptr)
            elem_data[4] = elem.parent()->id();
          else
            elem_data[4] = DofObject::invalid_processor_id;

          conn_data.resize(n_nodes);

          for (unsigned int i=0; i<n_nodes; i++)
            conn_data[i] = elem.node_id(i);

          io.data_stream(&elem_data[0],
                         cast_int<unsigned int>(elem_data.size()),
                         cast_int<unsigned int>(elem_data.size()));

#ifdef LIBMESH_ENABLE_UNIQUE_ID
          largest_id_type unique_id = elem.unique_id();

          io.data(unique_id, "# unique id");
#endif

#ifdef LIBMESH_ENABLE_AMR
          unsigned int p_level = elem.p_level();

          io.data(p_level, "# p_level");
#endif
          io.data_stream(&conn_data[0],
                         cast_int<unsigned int>(conn_data.size()),
                         cast_int<unsigned int>(conn_data.size()));
        }
    }
}


void CheckpointIO::write_remote_elem (Xdr & io) const
{
  libmesh_assert (io.writing());

  // convenient reference to our mesh
  const MeshBase & mesh = MeshOutput<MeshBase>::mesh();

  // Find the number of remote_elem links
  dof_id_type n_remote_elem = 0;
  std::vector<dof_id_type> elem_ids;
  std::vector<unsigned short int> elem_sides;

  {
    MeshBase::const_element_iterator
      it  = mesh.elements_begin(),
      end = mesh.elements_end();

    for (; it != end; ++it)
      {
        const Elem & elem = **it;

        for (unsigned int n=0; n != elem.n_neighbors(); ++n)
          {
            if (elem.neighbor(n) == remote_elem)
              {
                elem_ids.push_back(elem.id());
                elem_sides.push_back(n);
                n_remote_elem++;
              }
          }
      }
  }

  io.data(n_remote_elem, "# n_remote_elem");

  if (n_remote_elem)
    {
      io.data_stream(&elem_ids[0], n_remote_elem);
      io.data_stream(&elem_sides[0], n_remote_elem);
    }
}



void CheckpointIO::write_bcs (Xdr & io) const
{
  libmesh_assert (io.writing());

  // convenient reference to our mesh
  const MeshBase & mesh = MeshOutput<MeshBase>::mesh();

  // and our boundary info object
  const BoundaryInfo & boundary_info = mesh.get_boundary_info();

  // Version 0.9.2+ introduces entity names
  write_bc_names(io, boundary_info, true);  // sideset names
  write_bc_names(io, boundary_info, false); // nodeset names

  std::vector<dof_id_type> element_id_list;
  std::vector<unsigned short int> side_list;
  std::vector<boundary_id_type> bc_id_list;

  boundary_info.build_side_list(element_id_list, side_list, bc_id_list);

  io.data(element_id_list, "# element ids for bcs");
  io.data(side_list, "# sides of elements for bcs");
  io.data(bc_id_list, "# bc ids");
}



void CheckpointIO::write_nodesets (Xdr & io) const
{
  libmesh_assert (io.writing());

  // convenient reference to our mesh
  const MeshBase & mesh = MeshOutput<MeshBase>::mesh();

  // and our boundary info object
  const BoundaryInfo & boundary_info = mesh.get_boundary_info();

  std::vector<dof_id_type> node_id_list;
  std::vector<boundary_id_type> bc_id_list;

  boundary_info.build_node_list(node_id_list, bc_id_list);

  io.data(node_id_list, "# node id list");
  io.data(bc_id_list, "# nodeset bc id list");
}



void CheckpointIO::write_bc_names (Xdr & io, const BoundaryInfo & info, bool is_sideset) const
{
  const std::map<boundary_id_type, std::string> & boundary_map = is_sideset ?
    info.get_sideset_name_map() : info.get_nodeset_name_map();

  std::vector<boundary_id_type> boundary_ids;   boundary_ids.reserve(boundary_map.size());
  std::vector<std::string>  boundary_names; boundary_names.reserve(boundary_map.size());

  // We need to loop over the map and make sure that there aren't any invalid entries.  Since we
  // return writable references in boundary_info, it's possible for the user to leave some entity names
  // blank.  We can't write those to the XDA file.
  header_id_type n_boundary_names = 0;
  std::map<boundary_id_type, std::string>::const_iterator it_end = boundary_map.end();
  for (std::map<boundary_id_type, std::string>::const_iterator it = boundary_map.begin(); it != it_end; ++it)
    {
      if (!it->second.empty())
        {
          n_boundary_names++;
          boundary_ids.push_back(it->first);
          boundary_names.push_back(it->second);
        }
    }

  if (is_sideset)
    io.data(n_boundary_names, "# sideset id to name map");
  else
    io.data(n_boundary_names, "# nodeset id to name map");
  // Write out the ids and names in two vectors
  if (n_boundary_names)
    {
      io.data(boundary_ids);
      io.data(boundary_names);
    }
}



void CheckpointIO::read (const std::string & name)
{
  LOG_SCOPE("read()","CheckpointIO");

  MeshBase & mesh = MeshInput<MeshBase>::mesh();

  // Try to dynamic cast the mesh to see if it's a DistributedMesh object
  // Note: Just using is_serial() is not good enough because the Mesh won't
  // have been prepared yet when is when that flag gets set to false... sigh.
  _parallel = _parallel || !mesh.is_replicated();

  // If this is a serial read then we're going to only read the mesh
  // on processor 0, then broadcast it
  if ((_parallel && !mesh.is_replicated()) || mesh.processor_id() == 0)
    {
      // If we're trying to read a parallel checkpoint file on a
      // replicated mesh, we'll read every file on processor 0 so we
      // can broadcast it later.
      const processor_id_type begin_proc_id =
        (mesh.is_replicated() && _parallel) ? 0 : _my_processor_id;
      const processor_id_type end_proc_id =
        (mesh.is_replicated() && _parallel) ? _my_n_processors : (_my_processor_id+1);

      for (processor_id_type proc_id = begin_proc_id; proc_id != end_proc_id; ++proc_id)
        {
          std::ostringstream file_name_stream;

          file_name_stream << name;

          if (_parallel)
            file_name_stream << "-" << _my_n_processors << "-" << proc_id;

          {
            std::ifstream in (file_name_stream.str().c_str());

            if (!in.good())
              libmesh_error_msg("ERROR: cannot locate specified file:\n\t" << file_name_stream.str());
          }

          Xdr io (file_name_stream.str(), this->binary() ? DECODE : READ);

          // read the version
          io.data (_version);

          // read the dimension
          io.data (_mesh_dimension);
          mesh.set_mesh_dimension(_mesh_dimension);

          // Check if the mesh we're reading is the same as the one that was written
          {
            unsigned int parallel;
            io.data(parallel, "# parallel");

            if (_parallel != parallel)
              libmesh_error_msg("Attempted to read a " <<
                                (parallel ? "parallel" : "non-parallel")
                                << " (" << parallel << ')'
                                << " checkpoint file with a " <<
                                (_parallel ? "parallel" : "non-parallel")
                                << " checkpoint object!");
          }

          // If this is a parallel mesh then we need to check to ensure we're reading this on the same number of procs
          if (_parallel)
            {
              largest_id_type n_procs;
              io.data(n_procs, "# n_procs");

              if (n_procs != _my_n_processors)
                libmesh_error_msg("Attempted to utilize a checkpoint file on " << _my_n_processors << " processors but it was written using " << n_procs << "!!");
            }

          // read subdomain names
          this->read_subdomain_names(io);

          // read the nodal locations
          this->read_nodes (io);

          // read connectivity
          this->read_connectivity (io);

          // read the boundary conditions
          this->read_bcs (io);

          // read the nodesets
          this->read_nodesets (io);

          // read remote_elem connectivity
          this->read_remote_elem (io);

          io.close();
        }
    }

  // If the mesh was only read on processor 0 then we need to broadcast it
  if (mesh.is_replicated())
    MeshCommunication().broadcast(mesh);
  // If the mesh is distributed then we need to make sure it knows that
  else
    mesh.set_distributed();
}



void CheckpointIO::read_subdomain_names(Xdr & io)
{
  MeshBase & mesh = MeshInput<MeshBase>::mesh();

  std::map<subdomain_id_type, std::string> & subdomain_map =
    mesh.set_subdomain_name_map();

  std::vector<header_id_type> subdomain_ids;
  subdomain_ids.reserve(subdomain_map.size());

  std::vector<std::string>  subdomain_names;
  subdomain_names.reserve(subdomain_map.size());

  header_id_type n_subdomain_names = 0;<commit_msg>A one-processor mesh is never *really* distributed<commit_after>// The libMesh Finite Element Library.
// Copyright (C) 2002-2017 <NAME>, <NAME>, <NAME>

// This library is free software; you can redistribute it and/or
// modify it under the terms of the GNU Lesser General Public
// License as published by the Free Software Foundation; either
// version 2.1 of the License, or (at your option) any later version.

// This library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
// Lesser General Public License for more details.

// You should have received a copy of the GNU Lesser General Public
// License along with this library; if not, write to the Free Software
// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA

#include "libmesh/checkpoint_io.h"

// C++ includes
#include <iostream>
#include <iomanip>
#include <cstdio>
#include <vector>
#include <string>
#include <fstream>
#include <sstream> // for ostringstream

// Local includes
#include "libmesh/boundary_info.h"
#include "libmesh/distributed_mesh.h"
#include "libmesh/elem.h"
#include "libmesh/enum_xdr_mode.h"
#include "libmesh/libmesh_logging.h"
#include "libmesh/mesh_base.h"
#include "libmesh/mesh_communication.h"
#include "libmesh/mesh_tools.h"
#include "libmesh/node.h"
#include "libmesh/parallel.h"
#include "libmesh/partitioner.h"
#include "libmesh/remote_elem.h"
#include "libmesh/xdr_io.h"
#include "libmesh/xdr_cxx.h"

namespace libMesh
{

// ------------------------------------------------------------
// CheckpointIO members
CheckpointIO::CheckpointIO (MeshBase & mesh, const bool binary_in) :
  MeshInput<MeshBase> (mesh,/* is_parallel_format = */ true),
  MeshOutput<MeshBase>(mesh,/* is_parallel_format = */ true),
  ParallelObject      (mesh),
  _binary             (binary_in),
  _parallel           (false),
  _version            ("checkpoint-1.1"),
  _my_processor_id    (processor_id()),
  _my_n_processors    (n_processors())
{
}



CheckpointIO::CheckpointIO (const MeshBase & mesh, const bool binary_in) :
  MeshOutput<MeshBase>(mesh,/* is_parallel_format = */ true),
  ParallelObject      (mesh),
  _binary             (binary_in),
  _parallel           (false),
  _my_processor_id    (processor_id()),
  _my_n_processors    (n_processors())
{
}



CheckpointIO::~CheckpointIO ()
{
}




void CheckpointIO::write (const std::string & name)
{
  LOG_SCOPE("write()", "CheckpointIO");

  // convenient reference to our mesh
  const MeshBase & mesh = MeshOutput<MeshBase>::mesh();

  // Try to dynamic cast the mesh to see if it's a DistributedMesh object
  // Note: Just using is_serial() is not good enough because the Mesh won't
  // have been prepared yet when is when that flag gets set to false... sigh.
  _parallel = _parallel || !mesh.is_replicated();

  // If this is a serial mesh then we're only going to write it on processor 0
  if (_parallel || _my_processor_id == 0)
    {
      std::ostringstream file_name_stream;

      file_name_stream << name;

      if (_parallel)
        file_name_stream << "-" << _my_n_processors << "-" << _my_processor_id;

      Xdr io (file_name_stream.str(), this->binary() ? ENCODE : WRITE);

      // write the version
      io.data(_version, "# version");

      // Write out whether or not this is a serial mesh (helps with error checking on read)
      {
        unsigned int mesh_dimension = mesh.mesh_dimension();
        io.data(mesh_dimension, "# dimensions");
      }

      // Write out whether or not this is a serial mesh (helps with error checking on read)
      {
        unsigned int parallel = _parallel;
        io.data(parallel, "# parallel");
      }

      // If we're writing out a parallel mesh then we need to write the number of processors
      // so we can check it upon reading the file
      if (_parallel)
        {
          largest_id_type n_procs = _my_n_processors;
          io.data(n_procs, "# n_procs");
        }

      // write subdomain names
      this->write_subdomain_names(io);

      // write the nodal locations
      this->write_nodes (io);

      // write connectivity
      this->write_connectivity (io);

      // write the boundary condition information
      this->write_bcs (io);

      // write the nodeset information
      this->write_nodesets (io);

      // write remote_elem connectivity
      this->write_remote_elem (io);

      // close it up
      io.close();
    }

  // this->comm().barrier();
}

void CheckpointIO::write_subdomain_names(Xdr & io) const
{
  {
    const MeshBase & mesh = MeshOutput<MeshBase>::mesh();

    const std::map<subdomain_id_type, std::string> & subdomain_map = mesh.get_subdomain_name_map();

    std::vector<header_id_type> subdomain_ids;   subdomain_ids.reserve(subdomain_map.size());
    std::vector<std::string>  subdomain_names; subdomain_names.reserve(subdomain_map.size());

    // We need to loop over the map and make sure that there aren't any invalid entries.  Since we
    // return writable references in mesh_base, it's possible for the user to leave some entity names
    // blank.  We can't write those to the XDA file.
    header_id_type n_subdomain_names = 0;
    std::map<subdomain_id_type, std::string>::const_iterator it_end = subdomain_map.end();
    for (std::map<subdomain_id_type, std::string>::const_iterator it = subdomain_map.begin(); it != it_end; ++it)
      {
        if (!it->second.empty())
          {
            n_subdomain_names++;
            subdomain_ids.push_back(it->first);
            subdomain_names.push_back(it->second);
          }
      }

    io.data(n_subdomain_names, "# subdomain id to name map");
    // Write out the ids and names in two vectors
    if (n_subdomain_names)
      {
        io.data(subdomain_ids);
        io.data(subdomain_names);
      }
  }
}



void CheckpointIO::write_nodes (Xdr & io) const
{
  // convenient reference to our mesh
  const MeshBase & mesh = MeshOutput<MeshBase>::mesh();

  unsigned int n_nodes_here =
    std::distance(mesh.nodes_begin(), mesh.nodes_end());

  io.data(n_nodes_here, "# n_nodes on proc");

  // Will hold the node id and pid
  std::vector<largest_id_type> id_pid(2);

  // For the coordinates
  std::vector<Real> coords(LIBMESH_DIM);

  MeshBase::const_node_iterator
    it  = mesh.nodes_begin(),
    end = mesh.nodes_end();

  for (; it != end; ++it)
    {
      const Node & node = **it;

      id_pid[0] = node.id();
      id_pid[1] = node.processor_id();

      io.data_stream(&id_pid[0], 2, 2);

#ifdef LIBMESH_ENABLE_UNIQUE_ID
      largest_id_type unique_id = node.unique_id();

      io.data(unique_id, "# unique id");
#endif

      coords[0] = node(0);

#if LIBMESH_DIM > 1
      coords[1] = node(1);
#endif

#if LIBMESH_DIM > 2
      coords[2] = node(2);
#endif

      io.data_stream(&coords[0], LIBMESH_DIM, 3);
    }
}



void CheckpointIO::write_connectivity (Xdr & io) const
{
  libmesh_assert (io.writing());

  // convenient reference to our mesh
  const MeshBase & mesh = MeshOutput<MeshBase>::mesh();

  // We will only write active elements and their parents.
  unsigned int _n_active_levels = n_active_levels_on_processor(mesh);

  std::vector<xdr_id_type> n_elem_at_level(_n_active_levels, 0);

  // Find the number of elements at each level
  {
    MeshBase::const_element_iterator
      it  = mesh.elements_begin(),
      end = mesh.elements_end();

    for (; it != end; ++it)
      n_elem_at_level[(*it)->level()]++;
  }

  io.data(_n_active_levels, "# n_active_levels");

  // Put these out here to reduce memory churn
  // id type pid subdomain_id parent_id
  std::vector<largest_id_type> elem_data(5);
  std::vector<largest_id_type> conn_data;

  for (unsigned int level=0; level < _n_active_levels; level++)
    {
      std::ostringstream comment;
      comment << "# n_elem at level ";
      comment << level ;
      io.data (n_elem_at_level[level], comment.str().c_str());

      MeshBase::const_element_iterator
        it  = mesh.elements_begin(),
        end = mesh.elements_end();

      for (; it != end; ++it)
        {
          const Elem & elem = **it;

          // Only write elements of the current level.
          if (elem.level() != level)
            continue;

          unsigned int n_nodes = elem.n_nodes();

          elem_data[0] = elem.id();
          elem_data[1] = elem.type();
          elem_data[2] = elem.processor_id();
          elem_data[3] = elem.subdomain_id();

          if (elem.parent() != libmesh_nullptr)
            elem_data[4] = elem.parent()->id();
          else
            elem_data[4] = DofObject::invalid_processor_id;

          conn_data.resize(n_nodes);

          for (unsigned int i=0; i<n_nodes; i++)
            conn_data[i] = elem.node_id(i);

          io.data_stream(&elem_data[0],
                         cast_int<unsigned int>(elem_data.size()),
                         cast_int<unsigned int>(elem_data.size()));

#ifdef LIBMESH_ENABLE_UNIQUE_ID
          largest_id_type unique_id = elem.unique_id();

          io.data(unique_id, "# unique id");
#endif

#ifdef LIBMESH_ENABLE_AMR
          unsigned int p_level = elem.p_level();

          io.data(p_level, "# p_level");
#endif
          io.data_stream(&conn_data[0],
                         cast_int<unsigned int>(conn_data.size()),
                         cast_int<unsigned int>(conn_data.size()));
        }
    }
}


void CheckpointIO::write_remote_elem (Xdr & io) const
{
  libmesh_assert (io.writing());

  // convenient reference to our mesh
  const MeshBase & mesh = MeshOutput<MeshBase>::mesh();

  // Find the number of remote_elem links
  dof_id_type n_remote_elem = 0;
  std::vector<dof_id_type> elem_ids;
  std::vector<unsigned short int> elem_sides;

  {
    MeshBase::const_element_iterator
      it  = mesh.elements_begin(),
      end = mesh.elements_end();

    for (; it != end; ++it)
      {
        const Elem & elem = **it;

        for (unsigned int n=0; n != elem.n_neighbors(); ++n)
          {
            if (elem.neighbor(n) == remote_elem)
              {
                elem_ids.push_back(elem.id());
                elem_sides.push_back(n);
                n_remote_elem++;
              }
          }
      }
  }

  io.data(n_remote_elem, "# n_remote_elem");

  if (n_remote_elem)
    {
      io.data_stream(&elem_ids[0], n_remote_elem);
      io.data_stream(&elem_sides[0], n_remote_elem);
    }
}



void CheckpointIO::write_bcs (Xdr & io) const
{
  libmesh_assert (io.writing());

  // convenient reference to our mesh
  const MeshBase & mesh = MeshOutput<MeshBase>::mesh();

  // and our boundary info object
  const BoundaryInfo & boundary_info = mesh.get_boundary_info();

  // Version 0.9.2+ introduces entity names
  write_bc_names(io, boundary_info, true);  // sideset names
  write_bc_names(io, boundary_info, false); // nodeset names

  std::vector<dof_id_type> element_id_list;
  std::vector<unsigned short int> side_list;
  std::vector<boundary_id_type> bc_id_list;

  boundary_info.build_side_list(element_id_list, side_list, bc_id_list);

  io.data(element_id_list, "# element ids for bcs");
  io.data(side_list, "# sides of elements for bcs");
  io.data(bc_id_list, "# bc ids");
}



void CheckpointIO::write_nodesets (Xdr & io) const
{
  libmesh_assert (io.writing());

  // convenient reference to our mesh
  const MeshBase & mesh = MeshOutput<MeshBase>::mesh();

  // and our boundary info object
  const BoundaryInfo & boundary_info = mesh.get_boundary_info();

  std::vector<dof_id_type> node_id_list;
  std::vector<boundary_id_type> bc_id_list;

  boundary_info.build_node_list(node_id_list, bc_id_list);

  io.data(node_id_list, "# node id list");
  io.data(bc_id_list, "# nodeset bc id list");
}



void CheckpointIO::write_bc_names (Xdr & io, const BoundaryInfo & info, bool is_sideset) const
{
  const std::map<boundary_id_type, std::string> & boundary_map = is_sideset ?
    info.get_sideset_name_map() : info.get_nodeset_name_map();

  std::vector<boundary_id_type> boundary_ids;   boundary_ids.reserve(boundary_map.size());
  std::vector<std::string>  boundary_names; boundary_names.reserve(boundary_map.size());

  // We need to loop over the map and make sure that there aren't any invalid entries.  Since we
  // return writable references in boundary_info, it's possible for the user to leave some entity names
  // blank.  We can't write those to the XDA file.
  header_id_type n_boundary_names = 0;
  std::map<boundary_id_type, std::string>::const_iterator it_end = boundary_map.end();
  for (std::map<boundary_id_type, std::string>::const_iterator it = boundary_map.begin(); it != it_end; ++it)
    {
      if (!it->second.empty())
        {
          n_boundary_names++;
          boundary_ids.push_back(it->first);
          boundary_names.push_back(it->second);
        }
    }

  if (is_sideset)
    io.data(n_boundary_names, "# sideset id to name map");
  else
    io.data(n_boundary_names, "# nodeset id to name map");
  // Write out the ids and names in two vectors
  if (n_boundary_names)
    {
      io.data(boundary_ids);
      io.data(boundary_names);
    }
}



void CheckpointIO::read (const std::string & name)
{
  LOG_SCOPE("read()","CheckpointIO");

  MeshBase & mesh = MeshInput<MeshBase>::mesh();

  // Try to dynamic cast the mesh to see if it's a DistributedMesh object
  // Note: Just using is_serial() is not good enough because the Mesh won't
  // have been prepared yet when is when that flag gets set to false... sigh.
  _parallel = _parallel || !mesh.is_replicated();

  // If this is a serial read then we're going to only read the mesh
  // on processor 0, then broadcast it
  if ((_parallel && !mesh.is_replicated()) || mesh.processor_id() == 0)
    {
      // If we're trying to read a parallel checkpoint file on a
      // replicated mesh, we'll read every file on processor 0 so we
      // can broadcast it later.
      const processor_id_type begin_proc_id =
        (mesh.is_replicated() && _parallel) ? 0 : _my_processor_id;
      const processor_id_type end_proc_id =
        (mesh.is_replicated() && _parallel) ? _my_n_processors : (_my_processor_id+1);

      for (processor_id_type proc_id = begin_proc_id; proc_id != end_proc_id; ++proc_id)
        {
          std::ostringstream file_name_stream;

          file_name_stream << name;

          if (_parallel)
            file_name_stream << "-" << _my_n_processors << "-" << proc_id;

          {
            std::ifstream in (file_name_stream.str().c_str());

            if (!in.good())
              libmesh_error_msg("ERROR: cannot locate specified file:\n\t" << file_name_stream.str());
          }

          Xdr io (file_name_stream.str(), this->binary() ? DECODE : READ);

          // read the version
          io.data (_version);

          // read the dimension
          io.data (_mesh_dimension);
          mesh.set_mesh_dimension(_mesh_dimension);

          // Check if the mesh we're reading is the same as the one that was written
          {
            unsigned int parallel;
            io.data(parallel, "# parallel");

            if (_parallel != parallel)
              libmesh_error_msg("Attempted to read a " <<
                                (parallel ? "parallel" : "non-parallel")
                                << " (" << parallel << ')'
                                << " checkpoint file with a " <<
                                (_parallel ? "parallel" : "non-parallel")
                                << " checkpoint object!");
          }

          // If this is a parallel mesh then we need to check to ensure we're reading this on the same number of procs
          if (_parallel)
            {
              largest_id_type n_procs;
              io.data(n_procs, "# n_procs");

              if (n_procs != _my_n_processors)
                libmesh_error_msg("Attempted to utilize a checkpoint file on " << _my_n_processors << " processors but it was written using " << n_procs << "!!");
            }

          // read subdomain names
          this->read_subdomain_names(io);

          // read the nodal locations
          this->read_nodes (io);

          // read connectivity
          this->read_connectivity (io);

          // read the boundary conditions
          this->read_bcs (io);

          // read the nodesets
          this->read_nodesets (io);

          // read remote_elem connectivity
          this->read_remote_elem (io);

          io.close();
        }
    }

  // If the mesh was only read on processor 0 then we need to broadcast it
  if (mesh.is_replicated())
    MeshCommunication().broadcast(mesh);
  // If the mesh is really distributed then we need to make sure it
  // knows that
  else if (mesh.n_processors() > 1)
    mesh.set_distributed();
}



void CheckpointIO::read_subdomain_names(Xdr & io)
{
  MeshBase & mesh = MeshInput<MeshBase>::mesh();

  std::map<subdomain_id_type, std::string> & subdomain_map =
    mesh.set_subdomain_name_map();

  std::vector<header_id_type> subdomain_ids;
  subdomain_ids.reserve(subdomain_map.size());

  std::vector<std::string>  subdomain_names;
  subdomain_names.reserve(subdomain_map.size());

  header_id_type n_subdomain_names = 0;
{'fraction_non_alphanumeric': 0.12080673244341265, 'fraction_numerical': 0.004207777132907719, 'mean_word_length': 2.7364198200151795, 'pattern_counts': {'":': 0, '<': 205, '<?xml version=': 0, '>': 156, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 6, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3186565', 'n_tokens_mistral': 11205, 'n_tokens_neox': 10667, 'n_words': 2817}