<commit_before>#include "PvpLayer.hpp"
#include "arch/mpi/mpi.h"
#include "io/FileStream.hpp"
#include "io/fileio.hpp"
#include "structures/SparseList.hpp"

#include <cstring>
#include <iostream>

namespace PV {

PvpLayer::PvpLayer(const char *name, HyPerCol *hc) { initialize(name, hc); }

PvpLayer::~PvpLayer() {}

int PvpLayer::allocateDataStructures() {
   int status = InputLayer::allocateDataStructures();
   if (status != PV_SUCCESS) {
      return status;
   }
   if (parent->columnId() == 0) {
      FatalIf(
            getUsingFileList(),
            "%s: PvpLayer does not support using a list of files.\n",
            getName());
      initializeBatchIndexer(mPvpFrameCount);
   }
   return status;
}

Buffer<float> PvpLayer::retrieveData(std::string filename, int batchIndex) {

   // This is present so that when nextInput() is called during
   // InputLayer::allocateDataStructures, we correctly load the
   // inital state of the layer. Then, after InputLayer::allocate
   // is finished, PvpLayer::allocate reinitializes the BatchIndexer
   // so that the first update state does not skip the first
   // frame in the batch.
   if (mPvpFrameCount == -1) {
      FileStream headerStream(filename.c_str(), std::ios_base::in | std::ios_base::binary, false);
      struct BufferUtils::ActivityHeader header = BufferUtils::readActivityHeader(headerStream);

      mInputNx       = header.nx;
      mInputNy       = header.ny;
      mInputNf       = header.nf;
      mFileType      = header.fileType;
      mPvpFrameCount = header.nBands;
      initializeBatchIndexer(mPvpFrameCount);
      if (header.fileType == PVP_ACT_SPARSEVALUES_FILE_TYPE
          || header.fileType == PVP_ACT_FILE_TYPE) {
         sparseTable = BufferUtils::buildSparseFileTable(headerStream, mPvpFrameCount - 1);
      }
   }

   int frameNumber = 0;

   // If we're playing through the pvp file like a movie, use
   // BatchIndexer to get the frame number. Otherwise, just use
   // the start_frame_index value for this batch.
   if (getDisplayPeriod() > 0) {
      frameNumber = mBatchIndexer->nextIndex(batchIndex);
   }
   else {
      frameNumber = getStartIndex(batchIndex);
   }

   Buffer<float> result;
   BufferUtils::readActivityFromPvp<float>(filename.c_str(), &result, frameNumber);
   pvAssert(result.getWidth() == mInputNx);
   pvAssert(result.getHeight() == mInputNy);
   pvAssert(result.getFeatures() == mInputNf);

   return result;
}
}<commit_msg>Reverting change to PvpLayer that causes a huge slowdown when reading sparse pvp files<commit_after>#include "PvpLayer.hpp"
#include "arch/mpi/mpi.h"
#include "io/FileStream.hpp"
#include "io/fileio.hpp"
#include "structures/SparseList.hpp"

#include <cstring>
#include <iostream>

namespace PV {

PvpLayer::PvpLayer(const char *name, HyPerCol *hc) { initialize(name, hc); }

PvpLayer::~PvpLayer() {}

int PvpLayer::allocateDataStructures() {
   int status = InputLayer::allocateDataStructures();
   if (status != PV_SUCCESS) {
      return status;
   }
   if (parent->columnId() == 0) {
      FatalIf(
            getUsingFileList(),
            "%s: PvpLayer does not support using a list of files.\n",
            getName());
      initializeBatchIndexer(mPvpFrameCount);
   }
   return status;
}

Buffer<float> PvpLayer::retrieveData(std::string filename, int batchIndex) {

   // This is present so that when nextInput() is called during
   // InputLayer::allocateDataStructures, we correctly load the
   // inital state of the layer. Then, after InputLayer::allocate
   // is finished, PvpLayer::allocate reinitializes the BatchIndexer
   // so that the first update state does not skip the first
   // frame in the batch.
   if (mPvpFrameCount == -1) {
      FileStream headerStream(filename.c_str(), std::ios_base::in | std::ios_base::binary, false);
      struct BufferUtils::ActivityHeader header = BufferUtils::readActivityHeader(headerStream);

      mInputNx       = header.nx;
      mInputNy       = header.ny;
      mInputNf       = header.nf;
      mFileType      = header.fileType;
      mPvpFrameCount = header.nBands;
      initializeBatchIndexer(mPvpFrameCount);
      if (header.fileType == PVP_ACT_SPARSEVALUES_FILE_TYPE
          || header.fileType == PVP_ACT_FILE_TYPE) {
         sparseTable = BufferUtils::buildSparseFileTable(headerStream, mPvpFrameCount - 1);
      }
   }

   int frameNumber = 0;

   // If we're playing through the pvp file like a movie, use
   // BatchIndexer to get the frame number. Otherwise, just use
   // the start_frame_index value for this batch.
   if (getDisplayPeriod() > 0) {
      frameNumber = mBatchIndexer->nextIndex(batchIndex);
   }
   else {
      frameNumber = getStartIndex(batchIndex);
   }

   SparseList<float> list;
   Buffer<float> result(mInputNx, mInputNy, mInputNf);

   switch (mFileType) {
      case PVP_NONSPIKING_ACT_FILE_TYPE:
         BufferUtils::readDenseFromPvp<float>(filename.c_str(), &result, frameNumber);
         break;
      case PVP_ACT_SPARSEVALUES_FILE_TYPE:
         BufferUtils::readSparseFromPvp<float>(filename.c_str(), &list, frameNumber, &sparseTable);
         // This is a hack. We should only ever be
         // calling this with T == float.
         list.toBuffer(result, {0});
         break;
      case PVP_ACT_FILE_TYPE:
         // The {1} and {0} are the same hack.
         BufferUtils::readSparseBinaryFromPvp<float>(
               filename.c_str(), &list, frameNumber, {1}, &sparseTable);
         list.toBuffer(result, {0});
         break;
   }

   return result;
}
}
{'fraction_non_alphanumeric': 0.11205341032118368, 'fraction_numerical': 0.0027066041140382533, 'mean_word_length': 3.2572964669738864, 'pattern_counts': {'":': 0, '<': 16, '<?xml version=': 0, '>': 22, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 1, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7285015', 'n_tokens_mistral': 1773, 'n_tokens_neox': 1721, 'n_words': 426}