<commit_before>/****************************************************************/
/*               DO NOT MODIFY THIS HEADER                      */
/* MOOSE - Multiphysics Object Oriented Simulation Environment  */
/*                                                              */
/*           (c) 2010 Battelle Energy Alliance, LLC             */
/*                   ALL RIGHTS RESERVED                        */
/*                                                              */
/*          Prepared by Battelle Energy Alliance, LLC           */
/*            Under Contract No. DE-AC07-05ID14517              */
/*            With the U. S. Department of Energy               */
/*                                                              */
/*            See COPYRIGHT for full restrictions               */
/****************************************************************/

// STL includes
#include <sstream>
#include <iomanip>
#include <iterator>

// MOOSE includes
#include "DelimitedFileReader.h"
#include "MooseUtils.h"
#include "MooseError.h"
#include "pcrecpp.h"

namespace MooseUtils
{

DelimitedFileReader::DelimitedFileReader(const std::string & filename,
                                         const libMesh::Parallel::Communicator * comm)
  : _filename(filename),
    _header(HeaderFlag::AUTO),
    _ignore_empty_lines(true),
    _communicator(comm),
    _format("columns rows", "columns")
{
}

void
DelimitedFileReader::read()
{
  // Number of columns
  std::size_t n_cols;

  // Storage for the raw data
  std::vector<double> raw;
  std::size_t size_raw;

  // Read data
  if (_communicator == nullptr || _communicator->rank() == 0)
  {
    // Check the file
    MooseUtils::checkFileReadable(_filename);

    // Create the file stream
    std::ifstream stream_data(_filename);

    // Read/generate the header
    if (_format == "rows")
      readRowData(stream_data, raw);
    else
      readColumnData(stream_data, raw);

    // Set the number of columns
    n_cols = _names.size();

    // Close the stream
    stream_data.close();

    // Set raw data vector size
    size_raw = raw.size();
  }

  if (_communicator != nullptr)
  {
    // Broadcast column names
    _communicator->broadcast(n_cols);
    _names.resize(n_cols);
    _communicator->broadcast(_names);

    // Broadcast raw data
    _communicator->broadcast(size_raw);
    raw.resize(size_raw);
    _communicator->broadcast(raw);
  }

  // Resize the internal storage
  _data.resize(n_cols);

  // Process "row" formatted data
  if (_format == "rows")
  {
    std::vector<double>::iterator start = raw.begin();
    for (std::size_t j = 0; j < n_cols; ++j)
    {
      _data[j] = std::vector<double>(start, start + _row_offsets[j]);
      std::advance(start, _row_offsets[j]);
    }
  }

  // Process "column" formatted data
  else
  {
    mooseAssert(raw.size() % n_cols == 0,
                "The raw data is not evenly divisible by the number of columns.");
    const std::size_t n_rows = raw.size() / n_cols;
    for (std::size_t j = 0; j < n_cols; ++j)
    {
      _data[j].resize(n_rows);
      for (std::size_t i = 0; i < n_rows; ++i)
        _data[j][i] = raw[i * n_cols + j];
    }
  }<commit_msg>Communicate the row offsets when reading<commit_after>/****************************************************************/
/*               DO NOT MODIFY THIS HEADER                      */
/* MOOSE - Multiphysics Object Oriented Simulation Environment  */
/*                                                              */
/*           (c) 2010 Battelle Energy Alliance, LLC             */
/*                   ALL RIGHTS RESERVED                        */
/*                                                              */
/*          Prepared by Battelle Energy Alliance, LLC           */
/*            Under Contract No. DE-AC07-05ID14517              */
/*            With the U. S. Department of Energy               */
/*                                                              */
/*            See COPYRIGHT for full restrictions               */
/****************************************************************/

// STL includes
#include <sstream>
#include <iomanip>
#include <iterator>

// MOOSE includes
#include "DelimitedFileReader.h"
#include "MooseUtils.h"
#include "MooseError.h"
#include "pcrecpp.h"

namespace MooseUtils
{

DelimitedFileReader::DelimitedFileReader(const std::string & filename,
                                         const libMesh::Parallel::Communicator * comm)
  : _filename(filename),
    _header(HeaderFlag::AUTO),
    _ignore_empty_lines(true),
    _communicator(comm),
    _format("columns rows", "columns")
{
}

void
DelimitedFileReader::read()
{
  // Number of columns
  std::size_t n_cols;

  // Storage for the raw data
  std::vector<double> raw;
  std::size_t size_raw;
  std::size_t size_offsets;

  // Read data
  if (_communicator == nullptr || _communicator->rank() == 0)
  {
    // Check the file
    MooseUtils::checkFileReadable(_filename);

    // Create the file stream
    std::ifstream stream_data(_filename);

    // Read/generate the header
    if (_format == "rows")
      readRowData(stream_data, raw);
    else
      readColumnData(stream_data, raw);

    // Set the number of columns
    n_cols = _names.size();

    // Close the stream
    stream_data.close();

    // Set raw data vector size
    size_raw = raw.size();
    size_offsets = _row_offsets.size();
  }

  if (_communicator != nullptr)
  {
    // Broadcast column names
    _communicator->broadcast(n_cols);
    _names.resize(n_cols);
    _communicator->broadcast(_names);

    // Broadcast raw data
    _communicator->broadcast(size_raw);
    raw.resize(size_raw);
    _communicator->broadcast(raw);

    // Broadcast row offsets
    if (_format == "rows")
    {
      _communicator->broadcast(size_offsets);
      _row_offsets.resize(size_offsets);
      _communicator->broadcast(_row_offsets);
    }
  }

  // Resize the internal storage
  _data.resize(n_cols);

  // Process "row" formatted data
  if (_format == "rows")
  {
    std::vector<double>::iterator start = raw.begin();
    for (std::size_t j = 0; j < n_cols; ++j)
    {
      _data[j] = std::vector<double>(start, start + _row_offsets[j]);
      std::advance(start, _row_offsets[j]);
    }
  }

  // Process "column" formatted data
  else
  {
    mooseAssert(raw.size() % n_cols == 0,
                "The raw data is not evenly divisible by the number of columns.");
    const std::size_t n_rows = raw.size() / n_cols;
    for (std::size_t j = 0; j < n_cols; ++j)
    {
      _data[j].resize(n_rows);
      for (std::size_t i = 0; i < n_rows; ++i)
        _data[j][i] = raw[i * n_cols + j];
    }
  }
{'fraction_non_alphanumeric': 0.15673981191222572, 'fraction_numerical': 0.005373936408419167, 'mean_word_length': 2.035795197100136, 'pattern_counts': {'":': 0, '<': 21, '<?xml version=': 0, '>': 27, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 16, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4999516', 'n_tokens_mistral': 2021, 'n_tokens_neox': 1919, 'n_words': 468}