<commit_before>#ifndef ALEPH_TOPOLOGY_IO_SPARSE_ADJACENCY_MATRIX_HH__
#define ALEPH_TOPOLOGY_IO_SPARSE_ADJACENCY_MATRIX_HH__

#include <aleph/utilities/String.hh>

// FIXME: remove after debugging
#include <iostream>

#include <fstream>
#include <set>
#include <string>
#include <stdexcept>
#include <unordered_map>
#include <unordered_set>
#include <vector>

namespace aleph
{

namespace topology
{

namespace io
{

class SparseAdjacencyMatrixReader
{
public:
  template <class SimplicialComplex> void operator()( const std::string& filenameMatrix,
                                                      const std::string& filenameIndicator,
                                                      std::vector<SimplicialComplex>& result )
  {
    using Simplex    = typename SimplicialComplex::ValueType;
    using VertexType = typename Simplex::VertexType;
    using Edge       = std::pair<VertexType, VertexType>;

    std::vector<Edge> edges;
    std::unordered_set<VertexType> vertices;

    {
      std::ifstream in( filenameMatrix );
      if( !in )
        throw std::runtime_error( "Unable to read input adjacency matrix file" );

      std::string line;

      while( std::getline( in, line ) )
      {
        using namespace aleph::utilities;

        auto tokens = split( line, _separator );

        if( tokens.size() == 2 )
        {
          auto u = convert<VertexType>( tokens.front() );
          auto v = convert<VertexType>( tokens.back()  );

          edges.push_back( std::make_pair(u, v) );

          vertices.insert( u );
          vertices.insert( v );
        }
        else
        {
          // TODO: throw error?
        }
      }

      std::cerr << __PRETTY_FUNCTION__ << ": Read " << edges.size() << " edges\n";
    }

    // Maps a node ID to a graph ID, i.e. yields the inex of the graph
    // that should contain the node. All indices are starting at 1 but
    // will be mapped to 0 later on.
    std::unordered_map<VertexType, VertexType> node_id_to_graph_id;

    // Stores *all* graph IDs. The set makes sense here because I want
    // to ensure that repeated calls to this function always yield the
    // same order.
    std::set<VertexType> graphIDs;

    {
      std::ifstream in( filenameIndicator );
      if( !in )
        throw std::runtime_error( "Unable to read input graph indicator file" );

      std::string line;
      VertexType nodeID = 1;

      while( std::getline( in, line ) )
      {
        using namespace aleph::utilities;
        line = trim( line );

        auto graphID                    = convert<VertexType>( line );
        node_id_to_graph_id[ nodeID++ ] = graphID;

        graphIDs.insert( graphID );
      }
    }

    // Maps a graph ID (arbitrary start point) to an index in the
    // vector.

    using IndexType = std::size_t;
    std::unordered_map<VertexType, IndexType> graph_id_to_index;

    {
      IndexType index = IndexType();

      for( auto&& id : graphIDs )
        graph_id_to_index[id] = index++;
    }

    // Create output ---------------------------------------------------
    //
    // Create the set of output graphs and distribute the edges among
    // them according to their graph ID. This function also does some
    // sanity checks in order to check input data consistency.

    result.clear();
    result.resize( graphIDs.size() );

    for( auto&& vertex : vertices )
    {
      auto&& id    = node_id_to_graph_id[vertex];
      auto&& index = graph_id_to_index[id];
      auto&& K     = result[index];

      K.push_back( Simplex( vertex ) );
    }

    for( auto&& edge : edges )
    {
      auto&& u   = edge.first;
      auto&& v   = edge.second;
      auto&& uID = node_id_to_graph_id[u];
      auto&& vID = node_id_to_graph_id[v];

      if( uID != vID )
        throw std::runtime_error( "Format error: an edge must not belong to multiple graphs" );

      auto&& index = graph_id_to_index[ uID ];
      auto&& K     = result[index];

      K.push_back( Simplex( {u,v} ) );
    }
  }

private:

  // TODO: make configurable
  std::string _separator = ",";
};

} // namespace io

} // namespace topology

} // namespace aleph

#endif
<commit_msg>Prepared automated generation of relevant filenames<commit_after>#ifndef ALEPH_TOPOLOGY_IO_SPARSE_ADJACENCY_MATRIX_HH__
#define ALEPH_TOPOLOGY_IO_SPARSE_ADJACENCY_MATRIX_HH__

#include <aleph/utilities/Filesystem.hh>
#include <aleph/utilities/String.hh>

// FIXME: remove after debugging
#include <iostream>

#include <fstream>
#include <set>
#include <string>
#include <stdexcept>
#include <unordered_map>
#include <unordered_set>
#include <vector>

namespace aleph
{

namespace topology
{

namespace io
{

class SparseAdjacencyMatrixReader
{
public:
  template <class SimplicialComplex> void operator()( const std::string& filenameMatrix,
                                                      const std::string& filenameIndicator,
                                                      std::vector<SimplicialComplex>& result )
  {
    using Simplex    = typename SimplicialComplex::ValueType;
    using VertexType = typename Simplex::VertexType;
    using Edge       = std::pair<VertexType, VertexType>;

    std::vector<Edge> edges;
    std::unordered_set<VertexType> vertices;

    {
      std::ifstream in( filenameMatrix );
      if( !in )
        throw std::runtime_error( "Unable to read input adjacency matrix file" );

      std::string line;

      while( std::getline( in, line ) )
      {
        using namespace aleph::utilities;

        auto tokens = split( line, _separator );

        if( tokens.size() == 2 )
        {
          auto u = convert<VertexType>( tokens.front() );
          auto v = convert<VertexType>( tokens.back()  );

          edges.push_back( std::make_pair(u, v) );

          vertices.insert( u );
          vertices.insert( v );
        }
        else
        {
          // TODO: throw error?
        }
      }

      std::cerr << __PRETTY_FUNCTION__ << ": Read " << edges.size() << " edges\n";
    }

    // Maps a node ID to a graph ID, i.e. yields the inex of the graph
    // that should contain the node. All indices are starting at 1 but
    // will be mapped to 0 later on.
    std::unordered_map<VertexType, VertexType> node_id_to_graph_id;

    // Stores *all* graph IDs. The set makes sense here because I want
    // to ensure that repeated calls to this function always yield the
    // same order.
    std::set<VertexType> graphIDs;

    {
      std::ifstream in( filenameIndicator );
      if( !in )
        throw std::runtime_error( "Unable to read input graph indicator file" );

      std::string line;
      VertexType nodeID = 1;

      while( std::getline( in, line ) )
      {
        using namespace aleph::utilities;
        line = trim( line );

        auto graphID                    = convert<VertexType>( line );
        node_id_to_graph_id[ nodeID++ ] = graphID;

        graphIDs.insert( graphID );
      }
    }

    // Maps a graph ID (arbitrary start point) to an index in the
    // vector.

    using IndexType = std::size_t;
    std::unordered_map<VertexType, IndexType> graph_id_to_index;

    {
      IndexType index = IndexType();

      for( auto&& id : graphIDs )
        graph_id_to_index[id] = index++;
    }

    // Create output ---------------------------------------------------
    //
    // Create the set of output graphs and distribute the edges among
    // them according to their graph ID. This function also does some
    // sanity checks in order to check input data consistency.

    result.clear();
    result.resize( graphIDs.size() );

    for( auto&& vertex : vertices )
    {
      auto&& id    = node_id_to_graph_id[vertex];
      auto&& index = graph_id_to_index[id];
      auto&& K     = result[index];

      K.push_back( Simplex( vertex ) );
    }

    for( auto&& edge : edges )
    {
      auto&& u   = edge.first;
      auto&& v   = edge.second;
      auto&& uID = node_id_to_graph_id[u];
      auto&& vID = node_id_to_graph_id[v];

      if( uID != vID )
        throw std::runtime_error( "Format error: an edge must not belong to multiple graphs" );

      auto&& index = graph_id_to_index[ uID ];
      auto&& K     = result[index];

      K.push_back( Simplex( {u,v} ) );
    }

    std::cerr << getFilenameGraphIndicator( filenameMatrix ) << "\n";
  }

private:

  /**
   Given a base filename, gets its prefix. The prefix is everything that
   comes before the last `_` character. It is used to generate filenames
   for various auxiliary information about the graph if not specified by
   the client.
  */

  static std::string getPrefix( const std::string& filename )
  {
    using namespace aleph::utilities;

    auto s      = stem( filename );
    auto prefix = s.substr( 0, s.find_last_of( '_' ) );

    return prefix;
  }

  /*
    Given a base filename, calculates the default filename for the graph
    indicator values, for example. This filename is generated by getting
    a prefix of the filename and attaching e.g. `_graph_indicator.txt`.

    This function is *not* used if the user specified a file manually in
    which to find such information.
  */

  static std::string getFilenameGraphIndicator( const std::string& filename )  { return getPrefix(filename) + "_graph_indicator.txt";  }
  static std::string getFilenameGraphLabels( const std::string& filename )     { return getPrefix(filename) + "_graph_labels.txt";     }
  static std::string getFilenameNodeLabels( const std::string& filename )      { return getPrefix(filename) + "_node_labels.txt";      }
  static std::string getFilenameEdgeLabels( const std::string& filename )      { return getPrefix(filename) + "_edge_labels.txt";      }
  static std::string getFilenameEdgeAttributes( const std::string& filename )  { return getPrefix(filename) + "_edge_attributes.txt";  }
  static std::string getFilenameNodeAttributes( const std::string& filename )  { return getPrefix(filename) + "_node_attributes.txt";  }
  static std::string getFilenameGraphAttributes( const std::string& filename ) { return getPrefix(filename) + "_graph_attributes.txt"; }

  // TODO: make configurable
  std::string _separator = ",";
};

} // namespace io

} // namespace topology

} // namespace aleph

#endif

{'fraction_non_alphanumeric': 0.11674631815078514, 'fraction_numerical': 0.000877791865795377, 'mean_word_length': 2.4444071212630165, 'pattern_counts': {'":': 2, '<': 64, '<?xml version=': 0, '>': 44, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 8, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3743127', 'n_tokens_mistral': 3094, 'n_tokens_neox': 2982, 'n_words': 888}